diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 8ce5370e922e069..248abd6e038c856 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -124,6 +124,7 @@ def run_cases_test(self, input: str, expected: str): actual = instructions.strip() + "\n\n " + labels.strip() self.assertEqual(actual.strip(), expected.strip()) + return lines def test_inst_no_args(self): input = """ @@ -1791,6 +1792,45 @@ def test_spilled_label(self): """ self.run_cases_test(input, output) + def test_start_frame_separate_from_handlers(self): + input = """ + spilled label(start_frame) { + RELOAD_STACK(); + goto handler; + } + + label(handler) { + SAVE_STACK(); + goto start_frame; + } + """ + output = """ + LABEL(handler) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + JUMP_TO_LABEL(start_frame); + } + """ + generated = self.run_cases_test(input, output) + handlers, start_frame = generated.split( + "#if !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME)\n" + ) + self.assertNotIn("\n LABEL(start_frame)\n", handlers) + self.assertIn("#ifndef _Py_CEVAL_START_FRAME", handlers) + expected = """ + LABEL(start_frame) + { + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(handler); + } + +#endif /* !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) */ +#undef TIER_ONE + """ + self.assertEqual(start_frame.strip(), expected.strip()) + def test_incorrect_spills(self): input1 = """ diff --git a/Makefile.pre.in b/Makefile.pre.in index 36515dbd146a1e3..4a2526b5e8b8149 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -132,6 +132,8 @@ PY_CORE_EXE_LDFLAGS:= $(if $(CONFIGURE_EXE_LDFLAGS), $(CONFIGURE_EXE_LDFLAGS) $( CFLAGS_ALIASING=@CFLAGS_ALIASING@ # Compilation flags only for ceval.c. CFLAGS_CEVAL=@CFLAGS_CEVAL@ +# Compilation flags only for ceval_tailcall.c. +CFLAGS_CEVAL_TAILCALL=@CFLAGS_CEVAL_TAILCALL@ # Machine-dependent subdirectories @@ -445,6 +447,7 @@ PYTHON_OBJS= \ Python/bltinmodule.o \ Python/brc.o \ Python/ceval.o \ + Python/ceval_tailcall.o \ Python/codecs.o \ Python/codegen.o \ Python/compile.o \ @@ -2198,6 +2201,13 @@ Python/ceval.o: \ $(srcdir)/Python/opcode_targets.h \ $(srcdir)/Python/record_functions.c.h +Python/ceval_tailcall.o: \ + $(srcdir)/Python/ceval.h \ + $(srcdir)/Python/ceval_macros.h \ + $(srcdir)/Python/generated_cases.c.h \ + $(srcdir)/Python/opcode_targets.h \ + $(srcdir)/Python/record_functions.c.h + Python/flowgraph.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h @@ -3204,6 +3214,9 @@ Python/dtoa.o: Python/dtoa.c Python/ceval.o: Python/ceval.c $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_CEVAL) -o $@ $< +Python/ceval_tailcall.o: Python/ceval_tailcall.c + $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_CEVAL_TAILCALL) -o $@ $< + # Run reindent on the library .PHONY: reindent reindent: @@ -3462,7 +3475,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo. MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h -MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h +MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Python/ceval_macros.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-21-57-44.gh-issue-154124.ZErMAp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-21-57-44.gh-issue-154124.ZErMAp.rst new file mode 100644 index 000000000000000..d5a651c77a5d793 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-21-57-44.gh-issue-154124.ZErMAp.rst @@ -0,0 +1 @@ +Reserve frame pointer on tail calling builds using newer Clang. This recovers most of the performance hit from frame pointers on tail calling builds using recent versions of Clang diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index a67ecdc211bb343..a1d14068dc3d8e1 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -30,8 +30,13 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) #endif #if _Py_TAIL_CALL_INTERP +#define _Py_CEVAL_OPCODE_TARGETS_DEFINE #include "test_targets.h" +#undef _Py_CEVAL_OPCODE_TARGETS_DEFINE #include "test_cases.c.h" +#define _Py_CEVAL_START_FRAME +#include "test_cases.c.h" +#undef _Py_CEVAL_START_FRAME #endif PyObject* _Py_HOT_FUNCTION diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 3bdc16437e2bc61..7e92e06e0e197d2 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -7,6 +7,7 @@ #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 +#ifndef _Py_CEVAL_START_FRAME #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS @@ -13773,33 +13774,6 @@ JUMP_TO_LABEL(error); JUMP_TO_LABEL(error); } - LABEL(start_frame) - { - #ifdef Py_DEBUG - assert(frame->stackpointer_valid == 1); - #endif - int too_deep = _Py_EnterRecursivePy(tstate); - if (too_deep) { - JUMP_TO_LABEL(exit_unwind_notrace); - } - DTRACE_FUNCTION_ENTRY(); - next_instr = frame->instr_ptr; - #ifdef Py_DEBUG - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - if (lltrace < 0) { - JUMP_TO_LABEL(exit_unwind); - } - frame->lltrace = lltrace; - assert(!_PyErr_Occurred(tstate)); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - #if _Py_TAIL_CALL_INTERP - int opcode; - #endif - DISPATCH(); - } - #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) Py_GCC_ATTRIBUTE((unused)) #endif @@ -13827,4 +13801,35 @@ JUMP_TO_LABEL(error); } /* END LABELS */ +#endif /* _Py_CEVAL_START_FRAME */ +#if !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) + + LABEL(start_frame) + { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + JUMP_TO_LABEL(exit_unwind_notrace); + } + DTRACE_FUNCTION_ENTRY(); + next_instr = frame->instr_ptr; + #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; + assert(!_PyErr_Occurred(tstate)); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + #if _Py_TAIL_CALL_INTERP + int opcode; + #endif + DISPATCH(); + } + +#endif /* !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) */ #undef TIER_ONE diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4b..f42531563f85768 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -518,252 +518,254 @@ static void *opcode_tracing_targets_table[256] = { }; #endif #else /* _Py_TAIL_CALL_INTERP */ -static py_tail_call_funcptr instruction_funcptr_handler_table[256]; +extern py_tail_call_funcptr instruction_funcptr_handler_table[256]; -static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; +extern py_tail_call_funcptr instruction_funcptr_tracing_table[256]; -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_USTR_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); +#ifdef _Py_CEVAL_OPCODE_TARGETS_DEFINE +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_USTR_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); + +PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", @@ -773,7 +775,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { +py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -1031,7 +1033,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; -static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { +py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, @@ -1289,4 +1291,5 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; +#endif /* _Py_CEVAL_OPCODE_TARGETS_DEFINE */ #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c2d8f64a72b1191..53ec585ac436086 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -626,6 +626,9 @@ /std:clatest %(AdditionalOptions) + + /std:clatest %(AdditionalOptions) + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 2536fc971700deb..8da1b797e3a0009 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1421,6 +1421,9 @@ Python + + Python + Python diff --git a/Python/ceval.c b/Python/ceval.c index b90de789accd44f..7209659c063d242 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1083,8 +1083,8 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #if _Py_TIER2 // 0 for success, -1 for error. -static int -stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) +int +_PyEval_StopTracingAndJit(PyThreadState *tstate, _PyInterpreterFrame *frame) { int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); int err = 0; @@ -1111,7 +1111,10 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) #if _Py_TAIL_CALL_INTERP #include "opcode_targets.h" +/* Give start_frame its own frame pointer, independently of the handlers. */ +#define _Py_CEVAL_START_FRAME #include "generated_cases.c.h" +#undef _Py_CEVAL_START_FRAME #endif @@ -1196,8 +1199,8 @@ int _PyEval_StoreName(PyThreadState *tstate, _PyStackRef v, PyObject *name, PyOb #endif #ifdef WITH_DTRACE -static void -dtrace_function_entry(_PyInterpreterFrame *frame) +void +_PyEval_DTraceFunctionEntry(_PyInterpreterFrame *frame) { const char *filename; const char *funcname; @@ -1211,8 +1214,8 @@ dtrace_function_entry(_PyInterpreterFrame *frame) PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); } -static void -dtrace_function_return(_PyInterpreterFrame *frame) +void +_PyEval_DTraceFunctionReturn(_PyInterpreterFrame *frame) { const char *filename; const char *funcname; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 36921e35533433d..721d66fd502bd45 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -329,18 +329,23 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define CONSTS() _PyFrame_GetCode(frame)->co_consts #define NAMES() _PyFrame_GetCode(frame)->co_names +#if _Py_TIER2 && !defined(Py_BUILD_CORE_MODULE) +int _PyEval_StopTracingAndJit(PyThreadState *, _PyInterpreterFrame *); +#define stop_tracing_and_jit _PyEval_StopTracingAndJit +#endif + #if defined(WITH_DTRACE) && !defined(Py_BUILD_CORE_MODULE) -static void dtrace_function_entry(_PyInterpreterFrame *); -static void dtrace_function_return(_PyInterpreterFrame *); +void _PyEval_DTraceFunctionEntry(_PyInterpreterFrame *); +void _PyEval_DTraceFunctionReturn(_PyInterpreterFrame *); #define DTRACE_FUNCTION_ENTRY() \ if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ - dtrace_function_entry(frame); \ + _PyEval_DTraceFunctionEntry(frame); \ } #define DTRACE_FUNCTION_RETURN() \ if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ - dtrace_function_return(frame); \ + _PyEval_DTraceFunctionReturn(frame); \ } #else #define DTRACE_FUNCTION_ENTRY() ((void)0) diff --git a/Python/ceval_tailcall.c b/Python/ceval_tailcall.c new file mode 100644 index 000000000000000..66debb96b3c4575 --- /dev/null +++ b/Python/ceval_tailcall.c @@ -0,0 +1,13 @@ +#include "pyconfig.h" // _Py_TAIL_CALL_INTERP + +#if _Py_TAIL_CALL_INTERP +#include "ceval.h" +#include "pycore_long.h" + +#include "ceval_macros.h" + +#define _Py_CEVAL_OPCODE_TARGETS_DEFINE +#include "opcode_targets.h" +#undef _Py_CEVAL_OPCODE_TARGETS_DEFINE +#include "generated_cases.c.h" +#endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dd0ce41e4b06b4a..94aee7d8254a9bd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7,6 +7,7 @@ #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 +#ifndef _Py_CEVAL_START_FRAME #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS @@ -13770,33 +13771,6 @@ JUMP_TO_LABEL(error); JUMP_TO_LABEL(error); } - LABEL(start_frame) - { - #ifdef Py_DEBUG - assert(frame->stackpointer_valid == 1); - #endif - int too_deep = _Py_EnterRecursivePy(tstate); - if (too_deep) { - JUMP_TO_LABEL(exit_unwind_notrace); - } - DTRACE_FUNCTION_ENTRY(); - next_instr = frame->instr_ptr; - #ifdef Py_DEBUG - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - if (lltrace < 0) { - JUMP_TO_LABEL(exit_unwind); - } - frame->lltrace = lltrace; - assert(!_PyErr_Occurred(tstate)); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - #if _Py_TAIL_CALL_INTERP - int opcode; - #endif - DISPATCH(); - } - #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) Py_GCC_ATTRIBUTE((unused)) #endif @@ -13824,4 +13798,35 @@ JUMP_TO_LABEL(error); } /* END LABELS */ +#endif /* _Py_CEVAL_START_FRAME */ +#if !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) + + LABEL(start_frame) + { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + JUMP_TO_LABEL(exit_unwind_notrace); + } + DTRACE_FUNCTION_ENTRY(); + next_instr = frame->instr_ptr; + #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; + assert(!_PyErr_Occurred(tstate)); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + #if _Py_TAIL_CALL_INTERP + int opcode; + #endif + DISPATCH(); + } + +#endif /* !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) */ #undef TIER_ONE diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4b..f42531563f85768 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -518,252 +518,254 @@ static void *opcode_tracing_targets_table[256] = { }; #endif #else /* _Py_TAIL_CALL_INTERP */ -static py_tail_call_funcptr instruction_funcptr_handler_table[256]; +extern py_tail_call_funcptr instruction_funcptr_handler_table[256]; -static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; +extern py_tail_call_funcptr instruction_funcptr_tracing_table[256]; -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_USTR_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); +#ifdef _Py_CEVAL_OPCODE_TARGETS_DEFINE +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_USTR_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); + +PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", @@ -773,7 +775,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { +py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -1031,7 +1033,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; -static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { +py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, @@ -1289,4 +1291,5 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; +#endif /* _Py_CEVAL_OPCODE_TARGETS_DEFINE */ #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index e7cf6c223d22dc9..803229499391da4 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -43,19 +43,29 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: out.emit(f"#endif\n") out.emit("#else /* _Py_TAIL_CALL_INTERP */\n") -def function_proto(name: str) -> str: - return f"static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_{name}(TAIL_CALL_PARAMS)" +def function_signature(name: str) -> str: + return f"PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_{name}(TAIL_CALL_PARAMS)" + +def function_proto(name: str) -> str: + return f"extern {function_signature(name)}" def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: - out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") + shared_labels = ("error", "start_frame", "exit_unwind", "exit_unwind_notrace") + out.emit("extern py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") out.emit("\n") - out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") + out.emit("extern py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") out.emit("\n") + for name in shared_labels: + out.emit(f"{function_proto(name)};\n") + out.emit("\n") + out.emit("#ifdef _Py_CEVAL_OPCODE_TARGETS_DEFINE\n") + # Emit function prototypes for labels. for name in analysis.labels: - out.emit(f"{function_proto(name)};\n") + if name not in shared_labels: + out.emit(f"{function_proto(name)};\n") out.emit("\n") # Emit function prototypes for opcode handlers. @@ -64,7 +74,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit unknown opcode handler. - out.emit(function_proto("UNKNOWN_OPCODE")) + out.emit(function_signature("UNKNOWN_OPCODE")) out.emit(" {\n") out.emit("int opcode = next_instr->op.code;\n") out.emit(UNKNOWN_OPCODE_HANDLER) @@ -72,7 +82,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit the dispatch table. - out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") + out.emit("py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") named_values = analysis.opmap.values() @@ -82,7 +92,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("};\n") # Emit the tracing dispatch table. - out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") + out.emit("py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_TRACE_RECORD,\n") named_values = analysis.opmap.values() @@ -90,6 +100,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: if rest not in named_values: out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") out.emit("};\n") + out.emit("#endif /* _Py_CEVAL_OPCODE_TARGETS_DEFINE */\n") outfile.write("#endif /* _Py_TAIL_CALL_INTERP */\n") arg_parser = argparse.ArgumentParser( diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 0d477cb83c5d436..27792b4a945e05c 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -8,6 +8,7 @@ from analyzer import ( Analysis, Instruction, + Label, Uop, Part, analyze_files, @@ -149,6 +150,7 @@ def generate_tier1( #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 +#ifndef _Py_CEVAL_START_FRAME """) outfile.write(f""" #if !_Py_TAIL_CALL_INTERP @@ -187,27 +189,41 @@ def generate_tier1( emitter = Emitter(out, analysis.labels) generate_tier1_labels(analysis, emitter) outfile.write(f"{LABEL_END_MARKER}\n") + outfile.write("#endif /* _Py_CEVAL_START_FRAME */\n") + outfile.write("#if !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME)\n") + generate_tier1_start_frame(analysis, emitter) + outfile.write("#endif /* !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) */\n") outfile.write(FOOTER) -def generate_tier1_labels( - analysis: Analysis, emitter: Emitter -) -> None: +def generate_tier1_label(label: Label, emitter: Emitter) -> None: + if label.name == 'stop_tracing': + emitter.emit("#if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2)\n") + emitter.emit("Py_GCC_ATTRIBUTE((unused))\n") + emitter.emit("#endif\n") + emitter.emit(f"LABEL({label.name})\n") + storage = Storage(Stack(), [], [], 0, False) + if label.spilled: + storage.spilled = 1 + storage.stack.physical_sp = None + emitter.emit_tokens(label, storage, None) + emitter.emit("\n\n") + + +def generate_tier1_labels(analysis: Analysis, emitter: Emitter) -> None: emitter.emit("\n") # Emit tail-callable labels as function defintions - for name, label in analysis.labels.items(): - if name == 'stop_tracing': - emitter.emit("#if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2)\n") - emitter.emit("Py_GCC_ATTRIBUTE((unused))\n") - emitter.emit("#endif\n") - emitter.emit(f"LABEL({name})\n") - storage = Storage(Stack(), [], [], 0, False) - if label.spilled: - storage.spilled = 1 - storage.stack.physical_sp = None - emitter.emit_tokens(label, storage, None) - emitter.emit("\n\n") + for label in analysis.labels.values(): + if label.name != "start_frame": + generate_tier1_label(label, emitter) + + +def generate_tier1_start_frame(analysis: Analysis, emitter: Emitter) -> None: + emitter.emit("\n") + label = analysis.labels.get("start_frame") + if label is not None: + generate_tier1_label(label, emitter) def get_popped(inst: Instruction, analysis: Analysis) -> str: stack = get_stack_effect(inst) diff --git a/configure b/configure index c777d8117c66c4c..fd37ce0a4a5f28f 100755 --- a/configure +++ b/configure @@ -837,6 +837,7 @@ ENSUREPIP CFLAGS_CEVAL SRCDIRS THREADHEADERS +CFLAGS_CEVAL_TAILCALL PANEL_LIBS PANEL_CFLAGS CURSES_LIBS @@ -34396,6 +34397,9 @@ printf "%s\n" "#define HAVE_COMPUTED_GOTOS 1" >>confdefs.h esac +CFLAGS_CEVAL_TAILCALL="" + + # Check for --with-tail-call-interp { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-tail-call-interp" >&5 printf %s "checking for --with-tail-call-interp... " >&6; } @@ -34411,6 +34415,63 @@ printf "%s\n" "#define _Py_TAIL_CALL_INTERP 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } + reserve_frame_pointer_for_opcodes=no + if test -n "$frame_pointer_cflags" && test "x$with_frame_pointers" != xno; then + reserve_frame_pointer_for_opcodes=yes + fi + for flag in $CFLAGS; do + case $flag in #( + -fno-omit-frame-pointer|-fno-omit-leaf-frame-pointer|-mno-omit-leaf-frame-pointer) : + reserve_frame_pointer_for_opcodes=yes ;; #( + *) : + ;; +esac + done + if test "$reserve_frame_pointer_for_opcodes" = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved" >&5 +printf %s "checking whether C compiler accepts -fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved... " >&6; } +if test ${ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved=yes +else case e in #( + e) ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved" >&6; } +if test "x$ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved" = xyes +then : + + CFLAGS_CEVAL_TAILCALL="-fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved" + +else case e in #( + e) : ;; +esac +fi + + fi fi if test "$withval" = no then diff --git a/configure.ac b/configure.ac index a8760c94ac151aa..4c594b4982c3d5b 100644 --- a/configure.ac +++ b/configure.ac @@ -7692,6 +7692,9 @@ case "$ac_cv_computed_gotos" in yes*) [Define if the C compiler supports computed gotos.]) esac +CFLAGS_CEVAL_TAILCALL="" +AC_SUBST([CFLAGS_CEVAL_TAILCALL]) + # Check for --with-tail-call-interp AC_MSG_CHECKING([for --with-tail-call-interp]) AC_ARG_WITH( @@ -7706,6 +7709,22 @@ then AC_DEFINE([_Py_TAIL_CALL_INTERP], [1], [Define if you want to use tail-calling interpreters in CPython.]) AC_MSG_RESULT([yes]) + reserve_frame_pointer_for_opcodes=no + if test -n "$frame_pointer_cflags" && test "x$with_frame_pointers" != xno; then + reserve_frame_pointer_for_opcodes=yes + fi + dnl Also recognize frame pointers requested explicitly in CFLAGS. + for flag in $CFLAGS; do + AS_CASE([$flag], + [-fno-omit-frame-pointer|-fno-omit-leaf-frame-pointer|-mno-omit-leaf-frame-pointer], + [reserve_frame_pointer_for_opcodes=yes]) + done + if test "$reserve_frame_pointer_for_opcodes" = yes; then + dnl Reserve the register without adding a frame to each tail-call handler. + AX_CHECK_COMPILE_FLAG([-fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved], [ + CFLAGS_CEVAL_TAILCALL="-fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved" + ], [], [-Werror]) + fi fi if test "$withval" = no then