#include "Python.h"

#include "pycore_bitutils.h" // _Py_popcount32()

#include "pycore_call.h" // _PyObject_VectorcallTstate()

#include "pycore_ceval.h" // _PY_EVAL_EVENTS_BITS

#include "pycore_code.h" // _PyCode_Clear_Executors()

#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED()

#include "pycore_frame.h" // PyFrameObject

#include "pycore_interpframe.h" // _PyFrame_GetBytecode()

#include "pycore_long.h" // _PyLong_GetZero()

#include "pycore_modsupport.h" // _PyModule_CreateInitialized()

#include "pycore_namespace.h" // _PyNamespace_New()

#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE()

#include "pycore_opcode_utils.h" // IS_CONDITIONAL_JUMP_OPCODE()

#include "pycore_optimizer.h" // _PyExecutorObject

#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_UINTPTR_RELEASE()

#include "pycore_pystate.h" // _PyInterpreterState_GET()

#include "pycore_runtime_structs.h" // _PyCoMonitoringData

#include "pycore_tuple.h" // _PyTuple_FromArraySteal()

#include "opcode_ids.h"

/* Uncomment this to dump debugging output when assertions fail */

// #define INSTRUMENT_DEBUG 1

#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)

#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) \

if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \

_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); \

}

#define ASSERT_WORLD_STOPPED() assert(_PyInterpreterState_GET()->stoptheworld.world_stopped);

#else

#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj)

#define ASSERT_WORLD_STOPPED()

#endif

#ifdef Py_GIL_DISABLED

#define LOCK_CODE(code) \

assert(!_PyInterpreterState_GET()->stoptheworld.world_stopped); \

Py_BEGIN_CRITICAL_SECTION(code)

#define UNLOCK_CODE() Py_END_CRITICAL_SECTION()

#define MODIFY_BYTECODE(code, func, ...) \

do { \

PyCodeObject *co = (code); \

for (Py_ssize_t i = 0; i < code->co_tlbc->size; i++) { \

char *bc = co->co_tlbc->entries[i]; \

if (bc == NULL) { \

continue; \

} \

(func)(code, (_Py_CODEUNIT *)bc, __VA_ARGS__); \

} \

} while (0)

#else

#define LOCK_CODE(code)

#define UNLOCK_CODE()

#define MODIFY_BYTECODE(code, func, ...) \

(func)(code, _PyCode_CODE(code), __VA_ARGS__)

#endif

PyObject _PyInstrumentation_DISABLE = _PyObject_HEAD_INIT(&PyBaseObject_Type);

PyObject _PyInstrumentation_MISSING = _PyObject_HEAD_INIT(&PyBaseObject_Type);

static const int8_t EVENT_FOR_OPCODE[256] = {

[RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,

[INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,

[CALL] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL,

[CALL_KW] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_CALL_KW] = PY_MONITORING_EVENT_CALL,

[CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,

[LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,

[INSTRUMENTED_LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,

[RESUME] = -1,

[YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,

[INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,

[JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,

[JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,

[POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT,

[POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT,

[POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,

[POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,

[INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,

[INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,

[INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT,

[INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT,

[INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,

[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,

[FOR_ITER] = PY_MONITORING_EVENT_BRANCH_LEFT,

[INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH_LEFT,

[POP_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT,

[INSTRUMENTED_POP_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT,

[END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,

[INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,

[END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,

[INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,

[NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT,

[INSTRUMENTED_NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT,

[END_ASYNC_FOR] = PY_MONITORING_EVENT_BRANCH_RIGHT,

};

static const uint8_t DE_INSTRUMENT[256] = {

[INSTRUMENTED_RESUME] = RESUME,

[INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE,

[INSTRUMENTED_CALL] = CALL,

[INSTRUMENTED_CALL_KW] = CALL_KW,

[INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX,

[INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE,

[INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD,

[INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD,

[INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE,

[INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE,

[INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE,

[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE,

[INSTRUMENTED_FOR_ITER] = FOR_ITER,

[INSTRUMENTED_POP_ITER] = POP_ITER,

[INSTRUMENTED_END_FOR] = END_FOR,

[INSTRUMENTED_END_SEND] = END_SEND,

[INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR,

[INSTRUMENTED_NOT_TAKEN] = NOT_TAKEN,

[INSTRUMENTED_END_ASYNC_FOR] = END_ASYNC_FOR,

};

static const uint8_t INSTRUMENTED_OPCODES[256] = {

[RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,

[INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,

[CALL] = INSTRUMENTED_CALL,

[INSTRUMENTED_CALL] = INSTRUMENTED_CALL,

[CALL_KW] = INSTRUMENTED_CALL_KW,

[INSTRUMENTED_CALL_KW] = INSTRUMENTED_CALL_KW,

[CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,

[INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,

[YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,

[INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,

[RESUME] = INSTRUMENTED_RESUME,

[INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME,

[JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,

[INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,

[JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,

[INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,

[POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,

[INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,

[POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,

[INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,

[POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,

[INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,

[POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,

[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,

[END_FOR] = INSTRUMENTED_END_FOR,

[INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR,

[END_SEND] = INSTRUMENTED_END_SEND,

[INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND,

[FOR_ITER] = INSTRUMENTED_FOR_ITER,

[INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER,

[POP_ITER] = INSTRUMENTED_POP_ITER,

[INSTRUMENTED_POP_ITER] = INSTRUMENTED_POP_ITER,

[LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,

[INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,

[NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN,

[INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN,

[END_ASYNC_FOR] = INSTRUMENTED_END_ASYNC_FOR,

[INSTRUMENTED_END_ASYNC_FOR] = INSTRUMENTED_END_ASYNC_FOR,

[INSTRUMENTED_LINE] = INSTRUMENTED_LINE,

[INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION,

};

static inline bool

opcode_has_event(int opcode)

{

return (

opcode != INSTRUMENTED_LINE &&

INSTRUMENTED_OPCODES[opcode] > 0

);

}

static inline bool

is_instrumented(int opcode)

{

assert(opcode != 0);

assert(opcode != RESERVED);

return opcode < ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE;

}

#ifndef NDEBUG

static inline bool

monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b)

{

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

if (a.tools[i] != b.tools[i]) {

return false;

}

}

return true;

}

#endif

static inline _Py_LocalMonitors

monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b)

{

_Py_LocalMonitors res;

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

res.tools[i] = a.tools[i] & ~b.tools[i];

}

return res;

}

#ifndef NDEBUG

static inline _Py_LocalMonitors

monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b)

{

_Py_LocalMonitors res;

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

res.tools[i] = a.tools[i] & b.tools[i];

}

return res;

}

#endif

/* The union of the *local* events in a and b.

* Global events like RAISE are ignored.

* Used for instrumentation, as only local

* events get instrumented.

*/

static inline _Py_LocalMonitors

local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b)

{

_Py_LocalMonitors res;

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

res.tools[i] = a.tools[i] | b.tools[i];

}

return res;

}

static inline bool

monitors_are_empty(_Py_LocalMonitors m)

{

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

if (m.tools[i]) {

return false;

}

}

return true;

}

static inline bool

multiple_tools(_Py_LocalMonitors *m)

{

for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {

if (_Py_popcount32(m->tools[i]) > 1) {

return true;

}

}

return false;

}

static inline _PyMonitoringEventSet

get_local_events(_Py_LocalMonitors *m, int tool_id)

{

_PyMonitoringEventSet result = 0;

for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {

if ((m->tools[e] >> tool_id) & 1) {

result |= (1 << e);

}

}

return result;

}

static inline _PyMonitoringEventSet

get_events(_Py_GlobalMonitors *m, int tool_id)

{

_PyMonitoringEventSet result = 0;

for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {

if ((m->tools[e] >> tool_id) & 1) {

result |= (1 << e);

}

}

return result;

}

/* Module code can have line 0, even though modules start at line 1,

* so -1 is a legal delta. */

#define NO_LINE (-2)

/* Returns the line delta. Defined as:

* if line is None:

* line_delta = NO_LINE

* else:

* line_delta = line - first_line

*/

static int

compute_line_delta(PyCodeObject *code, int line)

{

if (line < 0) {

assert(line == -1);

return NO_LINE;

}

int delta = line - code->co_firstlineno;

assert(delta > NO_LINE);

return delta;

}

static int

compute_line(PyCodeObject *code, int line_delta)

{

if (line_delta == NO_LINE) {

return -1;

}

assert(line_delta > NO_LINE);

return code->co_firstlineno + line_delta;

}

int

_PyInstruction_GetLength(PyCodeObject *code, int offset)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

_Py_CODEUNIT inst = _Py_GetBaseCodeUnit(code, offset);

return 1 + _PyOpcode_Caches[inst.op.code];

}

static inline uint8_t

get_original_opcode(_PyCoLineInstrumentationData *line_data, int index)

{

return line_data->data[index*line_data->bytes_per_entry];

}

static inline uint8_t *

get_original_opcode_ptr(_PyCoLineInstrumentationData *line_data, int index)

{

return &line_data->data[index*line_data->bytes_per_entry];

}

static inline void

set_original_opcode(_PyCoLineInstrumentationData *line_data, int index, uint8_t opcode)

{

line_data->data[index*line_data->bytes_per_entry] = opcode;

}

static inline int

get_line_delta(_PyCoLineInstrumentationData *line_data, int index)

{

uint8_t *ptr = &line_data->data[index*line_data->bytes_per_entry+1];

assert(line_data->bytes_per_entry >= 2);

uint32_t value = *ptr;

for (int idx = 2; idx < line_data->bytes_per_entry; idx++) {

ptr++;

int shift = (idx-1)*8;

value |= ((uint32_t)(*ptr)) << shift;

}

assert(value < INT_MAX);

/* NO_LINE is stored as zero. */

return ((int)value) + NO_LINE;

}

static inline void

set_line_delta(_PyCoLineInstrumentationData *line_data, int index, int line_delta)

{

/* Store line_delta + 2 as we need -2 to represent no line number */

assert(line_delta >= NO_LINE);

uint32_t adjusted = line_delta - NO_LINE;

uint8_t *ptr = &line_data->data[index*line_data->bytes_per_entry+1];

assert(adjusted < (1ULL << ((line_data->bytes_per_entry-1)*8)));

assert(line_data->bytes_per_entry >= 2);

*ptr = adjusted & 0xff;

for (int idx = 2; idx < line_data->bytes_per_entry; idx++) {

ptr++;

adjusted >>= 8;

*ptr = adjusted & 0xff;

}

}

#ifdef INSTRUMENT_DEBUG

static void

dump_instrumentation_data_tools(PyCodeObject *code, uint8_t *tools, int i, FILE*out)

{

if (tools == NULL) {

fprintf(out, "tools = NULL");

}

else {

fprintf(out, "tools = %d", tools[i]);

}

}

static void

dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData *lines, int i, FILE*out)

{

if (lines == NULL) {

fprintf(out, ", lines = NULL");

}

else {

int opcode = get_original_opcode(lines, i);

int line_delta = get_line_delta(lines, i);

if (opcode == 0) {

fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", line_delta);

}

else {

fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[opcode], line_delta);

}

}

}

static void

dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, int i, FILE*out)

{

if (line_tools == NULL) {

fprintf(out, ", line_tools = NULL");

}

else {

fprintf(out, ", line_tools = %d", line_tools[i]);

}

}

static void

dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoMonitoringData *data, int i, FILE*out)

{

if (data->per_instruction_opcodes == NULL) {

fprintf(out, ", per-inst opcode = NULL");

}

else {

fprintf(out, ", per-inst opcode = %s", _PyOpcode_OpName[data->per_instruction_opcodes[i]]);

}

if (data->per_instruction_tools == NULL) {

fprintf(out, ", per-inst tools = NULL");

}

else {

fprintf(out, ", per-inst tools = %d", data->per_instruction_tools[i]);

}

}

static void

dump_global_monitors(const char *prefix, _Py_GlobalMonitors monitors, FILE*out)

{

fprintf(out, "%s monitors:\n", prefix);

for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) {

fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);

}

}

static void

dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out)

{

fprintf(out, "%s monitors:\n", prefix);

for (int event = 0; event < _PY_MONITORING_LOCAL_EVENTS; event++) {

fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);

}

}

/** NOTE:

* Do not use PyCode_Addr2Line to determine the line number in instrumentation,

* as `PyCode_Addr2Line` uses the monitoring data if it is available.

*/

/* No error checking -- Don't use this for anything but experimental debugging */

static void

dump_instrumentation_data(PyCodeObject *code, int star, FILE*out)

{

_PyCoMonitoringData *data = code->_co_monitoring;

fprintf(out, "\n");

PyObject_Print(code->co_name, out, Py_PRINT_RAW);

fprintf(out, "\n");

if (data == NULL) {

fprintf(out, "NULL\n");

return;

}

dump_global_monitors("Global", _PyInterpreterState_GET()->monitors, out);

dump_local_monitors("Code", data->local_monitors, out);

dump_local_monitors("Active", data->active_monitors, out);

int code_len = (int)Py_SIZE(code);

bool starred = false;

PyCodeAddressRange range;

_PyCode_InitAddressRange(code, &range);

for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) {

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

int opcode = instr->op.code;

if (i == star) {

fprintf(out, "** ");

starred = true;

}

fprintf(out, "Offset: %d, line: %d %s: ", i, _PyCode_CheckLineNumber(i*2, &range), _PyOpcode_OpName[opcode]);

dump_instrumentation_data_tools(code, data->tools, i, out);

dump_instrumentation_data_lines(code, data->lines, i, out);

dump_instrumentation_data_line_tools(code, data->line_tools, i, out);

dump_instrumentation_data_per_instruction(code, data, i, out);

fprintf(out, "\n");

;

}

if (!starred && star >= 0) {

fprintf(out, "Error offset not at valid instruction offset: %d\n", star);

fprintf(out, " ");

dump_instrumentation_data_tools(code, data->tools, star, out);

dump_instrumentation_data_lines(code, data->lines, star, out);

dump_instrumentation_data_line_tools(code, data->line_tools, star, out);

dump_instrumentation_data_per_instruction(code, data, star, out);

fprintf(out, "\n");

}

}

#define CHECK(test) do { \

if (!(test)) { \

dump_instrumentation_data(code, i, stderr); \

} \

assert(test); \

} while (0)

static bool

valid_opcode(int opcode)

{

if (opcode == INSTRUMENTED_LINE) {

return true;

}

if (IS_VALID_OPCODE(opcode) &&

opcode != CACHE &&

opcode != RESERVED &&

opcode < 254)

{

return true;

}

return false;

}

static void

sanity_check_instrumentation(PyCodeObject *code)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

_PyCoMonitoringData *data = code->_co_monitoring;

if (data == NULL) {

return;

}

_Py_GlobalMonitors global_monitors = _PyInterpreterState_GET()->monitors;

_Py_LocalMonitors active_monitors;

if (code->_co_monitoring) {

_Py_LocalMonitors local_monitors = code->_co_monitoring->local_monitors;

active_monitors = local_union(global_monitors, local_monitors);

}

else {

_Py_LocalMonitors empty = (_Py_LocalMonitors) { 0 };

active_monitors = local_union(global_monitors, empty);

}

assert(monitors_equals(

code->_co_monitoring->active_monitors,

active_monitors));

int code_len = (int)Py_SIZE(code);

PyCodeAddressRange range;

_PyCode_InitAddressRange(co, &range);

for (int i = 0; i < code_len;) {

_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];

int opcode = instr->op.code;

int base_opcode = _Py_GetBaseCodeUnit(code, i).op.code;

CHECK(valid_opcode(opcode));

CHECK(valid_opcode(base_opcode));

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode = data->per_instruction_opcodes[i];

if (!is_instrumented(opcode)) {

CHECK(_PyOpcode_Deopt[opcode] == opcode);

}

}

if (opcode == INSTRUMENTED_LINE) {

CHECK(data->lines);

opcode = get_original_opcode(data->lines, i);

CHECK(valid_opcode(opcode));

CHECK(opcode != END_FOR);

CHECK(opcode != RESUME);

CHECK(opcode != RESUME_CHECK);

CHECK(opcode != RESUME_CHECK_JIT);

CHECK(opcode != INSTRUMENTED_RESUME);

if (!is_instrumented(opcode)) {

CHECK(_PyOpcode_Deopt[opcode] == opcode);

}

CHECK(opcode != INSTRUMENTED_LINE);

}

else if (data->lines) {

/* If original_opcode is INSTRUMENTED_INSTRUCTION

* *and* we are executing a INSTRUMENTED_LINE instruction

* that has de-instrumented itself, then we will execute

* an invalid INSTRUMENTED_INSTRUCTION */

CHECK(get_original_opcode(data->lines, i) != INSTRUMENTED_INSTRUCTION);

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

CHECK(data->per_instruction_opcodes[i] != 0);

opcode = data->per_instruction_opcodes[i];

}

if (is_instrumented(opcode)) {

CHECK(DE_INSTRUMENT[opcode] == base_opcode);

int event = EVENT_FOR_OPCODE[DE_INSTRUMENT[opcode]];

if (event < 0) {

/* RESUME fixup */

event = instr->op.arg ? 1: 0;

}

CHECK(active_monitors.tools[event] != 0);

}

if (data->lines && get_original_opcode(data->lines, i)) {

int line1 = compute_line(code, get_line_delta(data->lines, i));

int line2 = _PyCode_CheckLineNumber(i*sizeof(_Py_CODEUNIT), &range);

CHECK(line1 == line2);

}

CHECK(valid_opcode(opcode));

if (data->tools) {

uint8_t local_tools = data->tools[i];

if (opcode_has_event(base_opcode)) {

int event = EVENT_FOR_OPCODE[base_opcode];

if (event == -1) {

/* RESUME fixup */

event = _PyCode_CODE(code)[i].op.arg;

}

CHECK((active_monitors.tools[event] & local_tools) == local_tools);

}

else {

CHECK(local_tools == 0xff);

}

}

i += _PyInstruction_GetLength(code, i);

assert(i <= code_len);

}

}

#else

#define CHECK(test) assert(test)

#endif

/* Get the underlying code unit, stripping instrumentation and ENTER_EXECUTOR */

_Py_CODEUNIT

_Py_GetBaseCodeUnit(PyCodeObject *code, int i)

{

_Py_CODEUNIT *src_instr = _PyCode_CODE(code) + i;

_Py_CODEUNIT inst = {

.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t *)src_instr)};

int opcode = inst.op.code;

if (opcode < MIN_INSTRUMENTED_OPCODE) {

inst.op.code = _PyOpcode_Deopt[opcode];

assert(inst.op.code < MIN_SPECIALIZED_OPCODE);

return inst;

}

if (opcode == ENTER_EXECUTOR) {

_PyExecutorObject *exec = code->co_executors->executors[inst.op.arg];

opcode = _PyOpcode_Deopt[exec->vm_data.opcode];

inst.op.code = opcode;

inst.op.arg = exec->vm_data.oparg;

assert(inst.op.code < MIN_SPECIALIZED_OPCODE);

return inst;

}

if (opcode == INSTRUMENTED_LINE) {

opcode = get_original_opcode(code->_co_monitoring->lines, i);

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode = code->_co_monitoring->per_instruction_opcodes[i];

}

CHECK(opcode != INSTRUMENTED_INSTRUCTION);

CHECK(opcode != INSTRUMENTED_LINE);

int deinstrumented = DE_INSTRUMENT[opcode];

if (deinstrumented) {

inst.op.code = deinstrumented;

}

else {

inst.op.code = _PyOpcode_Deopt[opcode];

}

assert(inst.op.code < MIN_SPECIALIZED_OPCODE);

return inst;

}

static void

de_instrument(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i,

int event)

{

assert(event != PY_MONITORING_EVENT_INSTRUCTION);

assert(event != PY_MONITORING_EVENT_LINE);

_Py_CODEUNIT *instr = &bytecode[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode = *opcode_ptr;

assert(opcode != ENTER_EXECUTOR);

if (opcode == INSTRUMENTED_LINE) {

opcode_ptr = get_original_opcode_ptr(monitoring->lines, i);

opcode = *opcode_ptr;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode_ptr = &monitoring->per_instruction_opcodes[i];

opcode = *opcode_ptr;

}

int deinstrumented = DE_INSTRUMENT[opcode];

if (deinstrumented == 0) {

return;

}

CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented);

FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented);

if (_PyOpcode_Caches[deinstrumented]) {

FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,

adaptive_counter_warmup().value_and_backoff);

}

}

static void

de_instrument_line(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring,

int i)

{

_Py_CODEUNIT *instr = &bytecode[i];

int opcode = instr->op.code;

if (opcode != INSTRUMENTED_LINE) {

return;

}

_PyCoLineInstrumentationData *lines = monitoring->lines;

int original_opcode = get_original_opcode(lines, i);

if (original_opcode == INSTRUMENTED_INSTRUCTION) {

set_original_opcode(lines, i, monitoring->per_instruction_opcodes[i]);

}

CHECK(original_opcode != 0);

CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);

FT_ATOMIC_STORE_UINT8(instr->op.code, original_opcode);

if (_PyOpcode_Caches[original_opcode]) {

FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,

adaptive_counter_warmup().value_and_backoff);

}

assert(instr->op.code != INSTRUMENTED_LINE);

}

static void

de_instrument_per_instruction(PyCodeObject *code, _Py_CODEUNIT *bytecode,

_PyCoMonitoringData *monitoring, int i)

{

_Py_CODEUNIT *instr = &bytecode[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode = *opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

opcode_ptr = get_original_opcode_ptr(monitoring->lines, i);

opcode = *opcode_ptr;

}

if (opcode != INSTRUMENTED_INSTRUCTION) {

return;

}

int original_opcode = monitoring->per_instruction_opcodes[i];

CHECK(original_opcode != 0);

CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);

FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, original_opcode);

if (_PyOpcode_Caches[original_opcode]) {

FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,

adaptive_counter_warmup().value_and_backoff);

}

assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION);

assert(instr->op.code != INSTRUMENTED_INSTRUCTION);

}

static void

instrument(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i)

{

_Py_CODEUNIT *instr = &bytecode[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode =*opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

opcode_ptr = get_original_opcode_ptr(monitoring->lines, i);

opcode = *opcode_ptr;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

opcode_ptr = &monitoring->per_instruction_opcodes[i];

opcode = *opcode_ptr;

CHECK(opcode != INSTRUMENTED_INSTRUCTION && opcode != INSTRUMENTED_LINE);

CHECK(opcode == _PyOpcode_Deopt[opcode]);

}

CHECK(opcode != 0);

if (!is_instrumented(opcode)) {

int deopt = _PyOpcode_Deopt[opcode];

int instrumented = INSTRUMENTED_OPCODES[deopt];

assert(instrumented);

FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented);

if (_PyOpcode_Caches[deopt]) {

FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,

adaptive_counter_warmup().value_and_backoff);

}

}

}

static void

instrument_line(PyCodeObject *code, _Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i)

{

uint8_t *opcode_ptr = &bytecode[i].op.code;

int opcode = *opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

return;

}

set_original_opcode(monitoring->lines, i, _PyOpcode_Deopt[opcode]);

CHECK(get_line_delta(monitoring->lines, i) > NO_LINE);

FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_LINE);

}

static void

instrument_per_instruction(PyCodeObject *code, _Py_CODEUNIT *bytecode,

_PyCoMonitoringData *monitoring, int i)

{

_Py_CODEUNIT *instr = &bytecode[i];

uint8_t *opcode_ptr = &instr->op.code;

int opcode = *opcode_ptr;

if (opcode == INSTRUMENTED_LINE) {

opcode_ptr = get_original_opcode_ptr(monitoring->lines, i);

opcode = *opcode_ptr;

}

if (opcode == INSTRUMENTED_INSTRUCTION) {

assert(monitoring->per_instruction_opcodes[i] > 0);

return;

}

CHECK(opcode != 0);

if (is_instrumented(opcode)) {

monitoring->per_instruction_opcodes[i] = opcode;

}

else {

assert(opcode != 0);

assert(_PyOpcode_Deopt[opcode] != 0);

assert(_PyOpcode_Deopt[opcode] != RESUME);

monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode];

}

assert(monitoring->per_instruction_opcodes[i] > 0);

FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_INSTRUCTION);

}

static void

remove_tools(PyCodeObject * code, int offset, int event, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

assert(event != PY_MONITORING_EVENT_LINE);

assert(event != PY_MONITORING_EVENT_INSTRUCTION);

assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));

assert(opcode_has_event(_Py_GetBaseCodeUnit(code, offset).op.code));

_PyCoMonitoringData *monitoring = code->_co_monitoring;

assert(monitoring);

bool should_de_instrument;

if (monitoring->tools) {

monitoring->tools[offset] &= ~tools;

should_de_instrument = (monitoring->tools[offset] == 0);

}

else {

/* Single tool */

uint8_t single_tool = monitoring->active_monitors.tools[event];

assert(_Py_popcount32(single_tool) <= 1);

should_de_instrument = ((single_tool & tools) == single_tool);

}

if (should_de_instrument) {

MODIFY_BYTECODE(code, de_instrument, monitoring, offset, event);

}

}

#ifndef NDEBUG

static bool

tools_is_subset_for_event(PyCodeObject * code, int event, int tools)

{

int global_tools = _PyInterpreterState_GET()->monitors.tools[event];

int local_tools = code->_co_monitoring->local_monitors.tools[event];

return tools == ((global_tools | local_tools) & tools);

}

#endif

static void

remove_line_tools(PyCodeObject * code, int offset, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

_PyCoMonitoringData *monitoring = code->_co_monitoring;

assert(monitoring);

bool should_de_instrument;

if (monitoring->line_tools)

{

uint8_t *toolsptr = &monitoring->line_tools[offset];

*toolsptr &= ~tools;

should_de_instrument = (*toolsptr == 0);

}

else {

/* Single tool */

uint8_t single_tool = monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE];

assert(_Py_popcount32(single_tool) <= 1);

should_de_instrument = ((single_tool & tools) == single_tool);

}

if (should_de_instrument) {

MODIFY_BYTECODE(code, de_instrument_line, monitoring, offset);

}

}

static void

add_tools(PyCodeObject * code, int offset, int event, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

assert(event != PY_MONITORING_EVENT_LINE);

assert(event != PY_MONITORING_EVENT_INSTRUCTION);

assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));

assert(code->_co_monitoring);

if (code->_co_monitoring &&

code->_co_monitoring->tools

) {

code->_co_monitoring->tools[offset] |= tools;

}

else {

/* Single tool */

assert(_Py_popcount32(tools) == 1);

assert(tools_is_subset_for_event(code, event, tools));

}

MODIFY_BYTECODE(code, instrument, code->_co_monitoring, offset);

}

static void

add_line_tools(PyCodeObject * code, int offset, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools));

assert(code->_co_monitoring);

if (code->_co_monitoring->line_tools) {

code->_co_monitoring->line_tools[offset] |= tools;

}

else {

/* Single tool */

assert(_Py_popcount32(tools) == 1);

}

MODIFY_BYTECODE(code, instrument_line, code->_co_monitoring, offset);

}

static void

add_per_instruction_tools(PyCodeObject * code, int offset, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools));

assert(code->_co_monitoring);

if (code->_co_monitoring->per_instruction_tools) {

code->_co_monitoring->per_instruction_tools[offset] |= tools;

}

else {

/* Single tool */

assert(_Py_popcount32(tools) == 1);

}

MODIFY_BYTECODE(code, instrument_per_instruction, code->_co_monitoring, offset);

}

static void

remove_per_instruction_tools(PyCodeObject * code, int offset, int tools)

{

ASSERT_WORLD_STOPPED_OR_LOCKED(code);

_PyCoMonitoringData *monitoring = code->_co_monitoring;

assert(code->_co_monitoring);

bool should_de_instrument;

if (code->_co_monitoring->per_instruction_tools) {

uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset];

*toolsptr &= ~tools;

should_de_instrument = (*toolsptr == 0);

}

else {

/* Single tool */

uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION];

assert(_Py_popcount32(single_tool) <= 1);

should_de_instrument = ((single_tool & tools) == single_tool);

}

if (should_de_instrument) {

MODIFY_BYTECODE(code, de_instrument_per_instruction, monitoring, offset);

}

}

/* Return 1 if DISABLE returned, -1 if error, 0 otherwise */

static int

call_one_instrument(

PyInterpreterState *interp, PyThreadState *tstate, PyObject **args,

size_t nargsf, int8_t tool, int event)

{

assert(0 <= tool && tool < 8);

assert(tstate->tracing == 0);

PyObject *instrument = interp->monitoring_callables[tool][event];

if (instrument == NULL) {

return 0;

}

int old_what = tstate->what_event;

tstate->what_event = event;

tstate->tracing++;

PyObject *res = _PyObject_VectorcallTstate(tstate, instrument, args, nargsf, NULL);

tstate->tracing--;

tstate->what_event = old_what;

if (res == NULL) {

return -1;

}

Py_DECREF(res);

return (res == &_PyInstrumentation_DISABLE);

}

static const int8_t MOST_SIGNIFICANT_BITS[16] = {

-1, 0, 1, 1,

2, 2, 2, 2,

3, 3, 3, 3,

3, 3, 3, 3,