From 792c23a2d2c459af39cd067a1796ee9416387de9 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sat, 18 Jul 2026 14:46:54 +0100 Subject: [PATCH 01/14] Meowl --- .../internal/pycore_ceval_opcode_targets.h | 70 +++ Makefile.pre.in | 12 + Modules/_testinternalcapi/interpreter.c | 6 +- Modules/_testinternalcapi/test_targets.h | 488 +++++++++--------- Python/ceval.c | 6 +- Python/ceval_macros.h | 53 -- Python/ceval_tailcall.c | 3 + Python/opcode_targets.h | 488 +++++++++--------- Tools/cases_generator/target_generator.py | 11 +- configure | 45 +- configure.ac | 24 + 11 files changed, 648 insertions(+), 558 deletions(-) create mode 100644 Include/internal/pycore_ceval_opcode_targets.h create mode 100644 Python/ceval_tailcall.c diff --git a/Include/internal/pycore_ceval_opcode_targets.h b/Include/internal/pycore_ceval_opcode_targets.h new file mode 100644 index 000000000000000..260ecb9dca37757 --- /dev/null +++ b/Include/internal/pycore_ceval_opcode_targets.h @@ -0,0 +1,70 @@ +#ifndef Py_INTERNAL_CEVAL_OPCODE_TARGETS_H +#define Py_INTERNAL_CEVAL_OPCODE_TARGETS_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#ifdef Py_STATS +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, int lastopcode +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, lastopcode +#else +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg +#endif + +#if _Py_TAIL_CALL_INTERP +# if defined(__clang__) || defined(__GNUC__) +# if !_Py__has_attribute(preserve_none) || !_Py__has_attribute(musttail) +# error "This compiler does not have support for efficient tail calling." +# endif +# elif defined(_MSC_VER) && (_MSC_VER < 1950) +# error "You need at least VS 2026 / PlatformToolset v145 for tail calling." +# endif +# if defined(_MSC_VER) && !defined(__clang__) +# define Py_MUSTTAIL [[msvc::musttail]] +# define Py_PRESERVE_NONE_CC __preserve_none +# else +# define Py_MUSTTAIL __attribute__((musttail)) +# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) +# endif +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table +# define TARGET(op) Py_NO_INLINE PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_##op(TAIL_CALL_PARAMS) + typedef PyObject *(Py_PRESERVE_NONE_CC *py_tail_call_funcptr)(TAIL_CALL_PARAMS); + +# define DISPATCH_GOTO() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define JUMP_TO_LABEL(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ + } while (0) +# ifdef Py_STATS +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, lastopcode); \ + } while (0) +# else +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg); \ + } while (0) +# endif +# define LABEL(name) TARGET(name) + #include "Python/opcode_targets.h" +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_CEVAL_OPCODE_TARGETS_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index b42a8491733c636..066e2632e361911 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -132,6 +132,8 @@ PY_CORE_EXE_LDFLAGS:= $(if $(CONFIGURE_EXE_LDFLAGS), $(CONFIGURE_EXE_LDFLAGS) $( CFLAGS_ALIASING=@CFLAGS_ALIASING@ # Compilation flags only for ceval.c. CFLAGS_CEVAL=@CFLAGS_CEVAL@ +# Compilation flags only for ceval_tailcall.c. +CFLAGS_CEVAL_TAILCALL=@CFLAGS_CEVAL@ # Machine-dependent subdirectories @@ -442,6 +444,7 @@ PYTHON_OBJS= \ Python/bltinmodule.o \ Python/brc.o \ Python/ceval.o \ + Python/ceval_tailcall.o \ Python/codecs.o \ Python/codegen.o \ Python/compile.o \ @@ -1310,6 +1313,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_cell.h \ $(srcdir)/Include/internal/pycore_ceval.h \ $(srcdir)/Include/internal/pycore_ceval_state.h \ + $(srcdir)/Include/internal/pycore_ceval_opcode_targets.h \ $(srcdir)/Include/internal/pycore_code.h \ $(srcdir)/Include/internal/pycore_codecs.h \ $(srcdir)/Include/internal/pycore_compile.h \ @@ -2208,6 +2212,11 @@ Python/ceval.o: \ $(srcdir)/Python/opcode_targets.h \ $(srcdir)/Python/record_functions.c.h +Python/ceval_tailcall.o: \ + $(srcdir)/Python/generated_cases.c.h \ + $(srcdir)/Python/opcode_targets.h \ + $(srcdir)/Python/record_functions.c.h + Python/flowgraph.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h @@ -3221,6 +3230,9 @@ Python/dtoa.o: Python/dtoa.c Python/ceval.o: Python/ceval.c $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_CEVAL) -o $@ $< +Python/ceval_tailcall.o: Python/ceval_tailcall.c + $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_CEVAL_TAILCALL) -o $@ $< + # Run reindent on the library .PHONY: reindent reindent: diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index 237635c5f2bcdba..4504f64a176907b 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -8,6 +8,7 @@ #include "../../Python/ceval.h" #include "../../Python/ceval_macros.h" +#include "pycore_ceval_opcode_targets.h" #undef IS_PEP523_HOOKED #define IS_PEP523_HOOKED(tstate) (tstate->interp->eval_frame != NULL && !tstate->interp->eval_frame_allow_specialization) @@ -31,11 +32,6 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) _PyJitEntryFuncPtr _Py_jit_entry; -#if _Py_TAIL_CALL_INTERP -#include "test_targets.h" -#include "test_cases.c.h" -#endif - PyObject* _Py_HOT_FUNCTION Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4b..6aca484e9e96cdc 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -518,252 +518,252 @@ static void *opcode_tracing_targets_table[256] = { }; #endif #else /* _Py_TAIL_CALL_INTERP */ -static py_tail_call_funcptr instruction_funcptr_handler_table[256]; +extern py_tail_call_funcptr instruction_funcptr_handler_table[256]; -static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; +extern py_tail_call_funcptr instruction_funcptr_tracing_table[256]; -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_USTR_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_USTR_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", @@ -773,7 +773,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { +extern py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -1031,7 +1031,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; -static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { +extern py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, diff --git a/Python/ceval.c b/Python/ceval.c index 3f636c3416e1207..11703f6de663436 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2,6 +2,7 @@ #include "ceval.h" #include "pycore_long.h" +#include "pycore_ceval_opcode_targets.h" int Py_GetRecursionLimit(void) @@ -1107,11 +1108,6 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) /* This setting is reversed below following _PyEval_EvalFrameDefault */ #endif -#if _Py_TAIL_CALL_INTERP -#include "opcode_targets.h" -#include "generated_cases.c.h" -#endif - _PyStackRef _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *index_or_null, int yield_from) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f19adfa0cfcfc15..c044dccdfe21ac6 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -71,60 +71,7 @@ #define INSTRUCTION_STATS(op) ((void)0) #endif -#ifdef Py_STATS -# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, int lastopcode -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, lastopcode -#else -# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg -#endif - #if _Py_TAIL_CALL_INTERP -# if defined(__clang__) || defined(__GNUC__) -# if !_Py__has_attribute(preserve_none) || !_Py__has_attribute(musttail) -# error "This compiler does not have support for efficient tail calling." -# endif -# elif defined(_MSC_VER) && (_MSC_VER < 1950) -# error "You need at least VS 2026 / PlatformToolset v145 for tail calling." -# endif -# if defined(_MSC_VER) && !defined(__clang__) -# define Py_MUSTTAIL [[msvc::musttail]] -# define Py_PRESERVE_NONE_CC __preserve_none -# else -# define Py_MUSTTAIL __attribute__((musttail)) -# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) -# endif - typedef PyObject *(Py_PRESERVE_NONE_CC *py_tail_call_funcptr)(TAIL_CALL_PARAMS); - -# define DISPATCH_TABLE_VAR instruction_funcptr_table -# define DISPATCH_TABLE instruction_funcptr_handler_table -# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table -# define TARGET(op) Py_NO_INLINE PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_##op(TAIL_CALL_PARAMS) - -# define DISPATCH_GOTO() \ - do { \ - Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ - } while (0) -# define DISPATCH_GOTO_NON_TRACING() \ - do { \ - Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ - } while (0) -# define JUMP_TO_LABEL(name) \ - do { \ - Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ - } while (0) -# ifdef Py_STATS -# define JUMP_TO_PREDICTED(name) \ - do { \ - Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, lastopcode); \ - } while (0) -# else -# define JUMP_TO_PREDICTED(name) \ - do { \ - Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg); \ - } while (0) -# endif -# define LABEL(name) TARGET(name) #elif USE_COMPUTED_GOTOS # define DISPATCH_TABLE_VAR opcode_targets # define DISPATCH_TABLE opcode_targets_table diff --git a/Python/ceval_tailcall.c b/Python/ceval_tailcall.c new file mode 100644 index 000000000000000..474528bb1e0a06c --- /dev/null +++ b/Python/ceval_tailcall.c @@ -0,0 +1,3 @@ +#if _Py_TAIL_CALL_INTERP +#include "generated_cases.c.h" +#endif \ No newline at end of file diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4b..6aca484e9e96cdc 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -518,252 +518,252 @@ static void *opcode_tracing_targets_table[256] = { }; #endif #else /* _Py_TAIL_CALL_INTERP */ -static py_tail_call_funcptr instruction_funcptr_handler_table[256]; +extern py_tail_call_funcptr instruction_funcptr_handler_table[256]; -static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; +extern py_tail_call_funcptr instruction_funcptr_tracing_table[256]; -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_USTR_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_USTR_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", @@ -773,7 +773,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { +extern py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -1031,7 +1031,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; -static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { +extern py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index e7cf6c223d22dc9..9ed253f7e3b0a6b 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -44,13 +44,12 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: out.emit("#else /* _Py_TAIL_CALL_INTERP */\n") def function_proto(name: str) -> str: - return f"static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_{name}(TAIL_CALL_PARAMS)" - + return f"extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_{name}(TAIL_CALL_PARAMS)" def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: - out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") + out.emit("extern py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") out.emit("\n") - out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") + out.emit("extern py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") out.emit("\n") # Emit function prototypes for labels. @@ -72,7 +71,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit the dispatch table. - out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") + out.emit("extern py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") named_values = analysis.opmap.values() @@ -82,7 +81,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("};\n") # Emit the tracing dispatch table. - out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") + out.emit("extern py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_TRACE_RECORD,\n") named_values = analysis.opmap.values() diff --git a/configure b/configure index c8494891087d045..f3909b3f6a5bb0d 100755 --- a/configure +++ b/configure @@ -835,9 +835,10 @@ OPENSSL_LDFLAGS OPENSSL_LIBS OPENSSL_INCLUDES ENSUREPIP -CFLAGS_CEVAL SRCDIRS THREADHEADERS +CFLAGS_CEVAL_TAILCALL +CFLAGS_CEVAL PANEL_LIBS PANEL_CFLAGS CURSES_LIBS @@ -31896,6 +31897,48 @@ printf "%s\n" "#define _Py_TAIL_CALL_INTERP 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } + if test -n "$frame_pointer_cflags" && test "x$with_frame_pointers" != xno; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we need to reserve the frame pointer in the opcode handlers" >&5 +printf %s "checking if we need to reserve the frame pointer in the opcode handlers... " >&6; } + if test "$cross_compiling" = yes +then : + reserve_frame_pointer_for_opcodes=undefined +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + int main(void) { + // clang-21 and newer support this CLI flag. + #if defined(__clang__) && (__clang_major__ >= 21) + return 1; + #else + return 0; + #endif + } + +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + reserve_frame_pointer_for_opcodes=no +else case e in #( + e) reserve_frame_pointer_for_opcodes=yes ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $reserve_frame_pointer_for_opcodes" >&5 +printf "%s\n" "$reserve_frame_pointer_for_opcodes" >&6; } + + if test "$reserve_frame_pointer_for_opcodes" = yes; then + CFLAGS_CEVAL_TAILCALL="-mreserve-frame-pointer-reg" + fi + + + + fi fi if test "$withval" = no then diff --git a/configure.ac b/configure.ac index ad8280349466e06..5c1548fa86be6e0 100644 --- a/configure.ac +++ b/configure.ac @@ -7445,6 +7445,30 @@ then AC_DEFINE([_Py_TAIL_CALL_INTERP], [1], [Define if you want to use tail-calling interpreters in CPython.]) AC_MSG_RESULT([yes]) + if test -n "$frame_pointer_cflags" && test "x$with_frame_pointers" != xno; then + AC_MSG_CHECKING([if we need to reserve the frame pointer in the opcode handlers]) + AC_RUN_IFELSE([AC_LANG_SOURCE([[ + int main(void) { + // clang-21 and newer support this CLI flag. + #if defined(__clang__) && (__clang_major__ >= 21) + return 1; + #else + return 0; + #endif + } + ]])], + [reserve_frame_pointer_for_opcodes=no], + [reserve_frame_pointer_for_opcodes=yes], + [reserve_frame_pointer_for_opcodes=undefined]) + AC_MSG_RESULT([$reserve_frame_pointer_for_opcodes]) + + if test "$reserve_frame_pointer_for_opcodes" = yes; then + CFLAGS_CEVAL_TAILCALL="-mreserve-frame-pointer-reg" + fi + AC_SUBST([CFLAGS_CEVAL]) + + AC_SUBST([CFLAGS_CEVAL_TAILCALL]) + fi fi if test "$withval" = no then From 50da138fb7d2cd3cbc226caad0619f5da4067b4a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 18 Jul 2026 15:11:16 +0100 Subject: [PATCH 02/14] GARBAGE --- .../internal/pycore_ceval_opcode_targets.h | 2 +- Makefile.pre.in | 9 ++++++-- Modules/_testinternalcapi/interpreter.c | 7 ++++++ Modules/_testinternalcapi/test_targets.h | 13 ++++++----- Python/ceval.c | 12 +++++----- Python/ceval_macros.h | 13 +++++++---- Python/ceval_tailcall.c | 10 ++++++++- Python/opcode_targets.h | 13 ++++++----- Tools/cases_generator/target_generator.py | 22 ++++++++++++++----- configure | 8 +++---- configure.ac | 6 ++--- 11 files changed, 79 insertions(+), 36 deletions(-) diff --git a/Include/internal/pycore_ceval_opcode_targets.h b/Include/internal/pycore_ceval_opcode_targets.h index 260ecb9dca37757..acf7f4bda30fd51 100644 --- a/Include/internal/pycore_ceval_opcode_targets.h +++ b/Include/internal/pycore_ceval_opcode_targets.h @@ -61,7 +61,7 @@ extern "C" { } while (0) # endif # define LABEL(name) TARGET(name) - #include "Python/opcode_targets.h" +# include "Python/opcode_targets.h" #endif #ifdef __cplusplus diff --git a/Makefile.pre.in b/Makefile.pre.in index 066e2632e361911..edf6d3619a4af46 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -133,7 +133,7 @@ CFLAGS_ALIASING=@CFLAGS_ALIASING@ # Compilation flags only for ceval.c. CFLAGS_CEVAL=@CFLAGS_CEVAL@ # Compilation flags only for ceval_tailcall.c. -CFLAGS_CEVAL_TAILCALL=@CFLAGS_CEVAL@ +CFLAGS_CEVAL_TAILCALL=@CFLAGS_CEVAL_TAILCALL@ # Machine-dependent subdirectories @@ -2213,10 +2213,15 @@ Python/ceval.o: \ $(srcdir)/Python/record_functions.c.h Python/ceval_tailcall.o: \ + $(srcdir)/Include/internal/pycore_ceval_opcode_targets.h \ + $(srcdir)/Python/ceval.h \ + $(srcdir)/Python/ceval_macros.h \ $(srcdir)/Python/generated_cases.c.h \ $(srcdir)/Python/opcode_targets.h \ $(srcdir)/Python/record_functions.c.h +Modules/_testinternalcapi/interpreter.o: PY_STDMODULE_CFLAGS += $(CFLAGS_CEVAL_TAILCALL) + Python/flowgraph.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h @@ -3491,7 +3496,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo. MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h -MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h +MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Include/internal/pycore_ceval_opcode_targets.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index 4504f64a176907b..cfc9867263a2312 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -32,6 +32,13 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) _PyJitEntryFuncPtr _Py_jit_entry; +#if _Py_TAIL_CALL_INTERP +#define _Py_CEVAL_OPCODE_TARGETS_DEFINE +#include "test_targets.h" +#undef _Py_CEVAL_OPCODE_TARGETS_DEFINE +#include "test_cases.c.h" +#endif + PyObject* _Py_HOT_FUNCTION Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 6aca484e9e96cdc..9518c53569fb5ea 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -522,13 +522,15 @@ extern py_tail_call_funcptr instruction_funcptr_handler_table[256]; extern py_tail_call_funcptr instruction_funcptr_tracing_table[256]; +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); + +#ifdef _Py_CEVAL_OPCODE_TARGETS_DEFINE extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); -extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); -extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); @@ -763,7 +765,7 @@ extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); -extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { +PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", @@ -773,7 +775,7 @@ extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -extern py_tail_call_funcptr instruction_funcptr_handler_table[256] = { +py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -1031,7 +1033,7 @@ extern py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; -extern py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { +py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, @@ -1289,4 +1291,5 @@ extern py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; +#endif /* _Py_CEVAL_OPCODE_TARGETS_DEFINE */ #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/ceval.c b/Python/ceval.c index 11703f6de663436..5f74a36cbd3bf02 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1082,8 +1082,8 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #if _Py_TIER2 // 0 for success, -1 for error. -static int -stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) +int +_PyEval_StopTracingAndJit(PyThreadState *tstate, _PyInterpreterFrame *frame) { int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); int err = 0; @@ -1190,8 +1190,8 @@ int _PyEval_StoreName(PyThreadState *tstate, _PyStackRef v, PyObject *name, PyOb #endif #ifdef WITH_DTRACE -static void -dtrace_function_entry(_PyInterpreterFrame *frame) +void +_PyEval_DTraceFunctionEntry(_PyInterpreterFrame *frame) { const char *filename; const char *funcname; @@ -1205,8 +1205,8 @@ dtrace_function_entry(_PyInterpreterFrame *frame) PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); } -static void -dtrace_function_return(_PyInterpreterFrame *frame) +void +_PyEval_DTraceFunctionReturn(_PyInterpreterFrame *frame) { const char *filename; const char *funcname; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c044dccdfe21ac6..556d401d8d886f8 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -276,18 +276,23 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define CONSTS() _PyFrame_GetCode(frame)->co_consts #define NAMES() _PyFrame_GetCode(frame)->co_names +#if _Py_TIER2 && !defined(Py_BUILD_CORE_MODULE) +int _PyEval_StopTracingAndJit(PyThreadState *, _PyInterpreterFrame *); +#define stop_tracing_and_jit _PyEval_StopTracingAndJit +#endif + #if defined(WITH_DTRACE) && !defined(Py_BUILD_CORE_MODULE) -static void dtrace_function_entry(_PyInterpreterFrame *); -static void dtrace_function_return(_PyInterpreterFrame *); +void _PyEval_DTraceFunctionEntry(_PyInterpreterFrame *); +void _PyEval_DTraceFunctionReturn(_PyInterpreterFrame *); #define DTRACE_FUNCTION_ENTRY() \ if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ - dtrace_function_entry(frame); \ + _PyEval_DTraceFunctionEntry(frame); \ } #define DTRACE_FUNCTION_RETURN() \ if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ - dtrace_function_return(frame); \ + _PyEval_DTraceFunctionReturn(frame); \ } #else #define DTRACE_FUNCTION_ENTRY() ((void)0) diff --git a/Python/ceval_tailcall.c b/Python/ceval_tailcall.c index 474528bb1e0a06c..566ee1ea8009437 100644 --- a/Python/ceval_tailcall.c +++ b/Python/ceval_tailcall.c @@ -1,3 +1,11 @@ +#include "ceval.h" +#include "pycore_long.h" + +#include "ceval_macros.h" + #if _Py_TAIL_CALL_INTERP +#define _Py_CEVAL_OPCODE_TARGETS_DEFINE +#include "pycore_ceval_opcode_targets.h" +#undef _Py_CEVAL_OPCODE_TARGETS_DEFINE #include "generated_cases.c.h" -#endif \ No newline at end of file +#endif diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 6aca484e9e96cdc..9518c53569fb5ea 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -522,13 +522,15 @@ extern py_tail_call_funcptr instruction_funcptr_handler_table[256]; extern py_tail_call_funcptr instruction_funcptr_tracing_table[256]; +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); + +#ifdef _Py_CEVAL_OPCODE_TARGETS_DEFINE extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); -extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); -extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); @@ -763,7 +765,7 @@ extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_C extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); -extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { +PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", @@ -773,7 +775,7 @@ extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -extern py_tail_call_funcptr instruction_funcptr_handler_table[256] = { +py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -1031,7 +1033,7 @@ extern py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; -extern py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { +py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, @@ -1289,4 +1291,5 @@ extern py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; +#endif /* _Py_CEVAL_OPCODE_TARGETS_DEFINE */ #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 9ed253f7e3b0a6b..aa3e6a46558f917 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -43,18 +43,29 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: out.emit(f"#endif\n") out.emit("#else /* _Py_TAIL_CALL_INTERP */\n") +def function_signature(name: str) -> str: + return f"PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_{name}(TAIL_CALL_PARAMS)" + + def function_proto(name: str) -> str: - return f"extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_{name}(TAIL_CALL_PARAMS)" + return f"extern {function_signature(name)}" def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: + shared_labels = ("error", "start_frame") out.emit("extern py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") out.emit("\n") out.emit("extern py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") out.emit("\n") + for name in shared_labels: + out.emit(f"{function_proto(name)};\n") + out.emit("\n") + out.emit("#ifdef _Py_CEVAL_OPCODE_TARGETS_DEFINE\n") + # Emit function prototypes for labels. for name in analysis.labels: - out.emit(f"{function_proto(name)};\n") + if name not in shared_labels: + out.emit(f"{function_proto(name)};\n") out.emit("\n") # Emit function prototypes for opcode handlers. @@ -63,7 +74,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit unknown opcode handler. - out.emit(function_proto("UNKNOWN_OPCODE")) + out.emit(function_signature("UNKNOWN_OPCODE")) out.emit(" {\n") out.emit("int opcode = next_instr->op.code;\n") out.emit(UNKNOWN_OPCODE_HANDLER) @@ -71,7 +82,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit the dispatch table. - out.emit("extern py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") + out.emit("py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") named_values = analysis.opmap.values() @@ -81,7 +92,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("};\n") # Emit the tracing dispatch table. - out.emit("extern py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") + out.emit("py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_TRACE_RECORD,\n") named_values = analysis.opmap.values() @@ -89,6 +100,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: if rest not in named_values: out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") out.emit("};\n") + out.emit("#endif /* _Py_CEVAL_OPCODE_TARGETS_DEFINE */\n") outfile.write("#endif /* _Py_TAIL_CALL_INTERP */\n") arg_parser = argparse.ArgumentParser( diff --git a/configure b/configure index f3909b3f6a5bb0d..9159dd2fac6e7ba 100755 --- a/configure +++ b/configure @@ -835,10 +835,10 @@ OPENSSL_LDFLAGS OPENSSL_LIBS OPENSSL_INCLUDES ENSUREPIP +CFLAGS_CEVAL SRCDIRS THREADHEADERS CFLAGS_CEVAL_TAILCALL -CFLAGS_CEVAL PANEL_LIBS PANEL_CFLAGS CURSES_LIBS @@ -31882,6 +31882,9 @@ printf "%s\n" "#define HAVE_COMPUTED_GOTOS 1" >>confdefs.h esac +CFLAGS_CEVAL_TAILCALL="" + + # Check for --with-tail-call-interp { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-tail-call-interp" >&5 printf %s "checking for --with-tail-call-interp... " >&6; } @@ -31935,9 +31938,6 @@ printf "%s\n" "$reserve_frame_pointer_for_opcodes" >&6; } if test "$reserve_frame_pointer_for_opcodes" = yes; then CFLAGS_CEVAL_TAILCALL="-mreserve-frame-pointer-reg" fi - - - fi fi if test "$withval" = no diff --git a/configure.ac b/configure.ac index 5c1548fa86be6e0..28e39571db19d44 100644 --- a/configure.ac +++ b/configure.ac @@ -7431,6 +7431,9 @@ case "$ac_cv_computed_gotos" in yes*) [Define if the C compiler supports computed gotos.]) esac +CFLAGS_CEVAL_TAILCALL="" +AC_SUBST([CFLAGS_CEVAL_TAILCALL]) + # Check for --with-tail-call-interp AC_MSG_CHECKING([for --with-tail-call-interp]) AC_ARG_WITH( @@ -7465,9 +7468,6 @@ then if test "$reserve_frame_pointer_for_opcodes" = yes; then CFLAGS_CEVAL_TAILCALL="-mreserve-frame-pointer-reg" fi - AC_SUBST([CFLAGS_CEVAL]) - - AC_SUBST([CFLAGS_CEVAL_TAILCALL]) fi fi if test "$withval" = no From f030c0b415e745411a7ae0baaf406b8df3411cc2 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 19 Jul 2026 13:44:18 +0100 Subject: [PATCH 03/14] Oiia --- Modules/_testinternalcapi/interpreter.c | 2 ++ Modules/_testinternalcapi/test_cases.c.h | 27 ---------------- Python/bytecodes.c | 29 ----------------- Python/ceval.c | 6 ++++ Python/ceval_macros.h | 41 ++++++++++++++++++++++++ Python/generated_cases.c.h | 27 ---------------- configure | 2 +- configure.ac | 2 +- 8 files changed, 51 insertions(+), 85 deletions(-) diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index cfc9867263a2312..d9e439725a558bf 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -37,6 +37,7 @@ _PyJitEntryFuncPtr _Py_jit_entry; #include "test_targets.h" #undef _Py_CEVAL_OPCODE_TARGETS_DEFINE #include "test_cases.c.h" +START_FRAME(); #endif PyObject* _Py_HOT_FUNCTION @@ -146,6 +147,7 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) # endif #else goto start_frame; + START_FRAME(); #include "test_cases.c.h" #endif diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index a17648a33d4fe4a..1bd1e815f2f3c95 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -13841,33 +13841,6 @@ JUMP_TO_LABEL(error); JUMP_TO_LABEL(error); } - LABEL(start_frame) - { - #ifdef Py_DEBUG - assert(frame->stackpointer_valid == 1); - #endif - int too_deep = _Py_EnterRecursivePy(tstate); - if (too_deep) { - JUMP_TO_LABEL(exit_unwind_notrace); - } - DTRACE_FUNCTION_ENTRY(); - next_instr = frame->instr_ptr; - #ifdef Py_DEBUG - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - if (lltrace < 0) { - JUMP_TO_LABEL(exit_unwind); - } - frame->lltrace = lltrace; - assert(!_PyErr_Occurred(tstate)); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - #if _Py_TAIL_CALL_INTERP - int opcode; - #endif - DISPATCH(); - } - #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) Py_GCC_ATTRIBUTE((unused)) #endif diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4d7b338e2dbd4c3..8efce3141a3cb58 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -6563,34 +6563,6 @@ dummy_func( goto error; } - spilled label(start_frame) { - #ifdef Py_DEBUG - assert(frame->stackpointer_valid == 1); - #endif - int too_deep = _Py_EnterRecursivePy(tstate); - if (too_deep) { - goto exit_unwind_notrace; - } - DTRACE_FUNCTION_ENTRY(); - next_instr = frame->instr_ptr; - #ifdef Py_DEBUG - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - if (lltrace < 0) { - JUMP_TO_LABEL(exit_unwind); - } - frame->lltrace = lltrace; - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); - #endif - RELOAD_STACK(); -#if _Py_TAIL_CALL_INTERP - int opcode; -#endif - DISPATCH(); - } - inst(TRACE_RECORD, (--)) { #if _Py_TIER2 assert(IS_JIT_TRACING()); @@ -6673,7 +6645,6 @@ dummy_func( exit_unwind_notrace: handle_eval_breaker: resume_frame: - start_frame: unbound_local_error: ; } diff --git a/Python/ceval.c b/Python/ceval.c index 5f74a36cbd3bf02..acbea8a0e3a6bba 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1221,6 +1221,11 @@ _PyEval_DTraceFunctionReturn(_PyInterpreterFrame *frame) } #endif +#if _Py_TAIL_CALL_INTERP +LABEL(exit_unwind_notrace); +START_FRAME(); +#endif + PyObject* _Py_HOT_FUNCTION DONT_SLP_VECTORIZE _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { @@ -1335,6 +1340,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int # endif #else goto start_frame; + START_FRAME(); # include "generated_cases.c.h" #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 556d401d8d886f8..fb9a532f02db426 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -613,3 +613,44 @@ gen_try_set_executing(PyGenObject *gen) #define CALL_TP_ITERITEM_NO_ESCAPE(ITER, INDEX) \ Py_TYPE(ITER)->_tp_iteritem((ITER), (INDEX)) + +#if Py_DEBUG +#define START_FRAME() \ + LABEL(start_frame) { \ + assert(frame->stackpointer_valid == 1); \ + int too_deep = _Py_EnterRecursivePy(tstate); \ + if (too_deep) { \ + JUMP_TO_LABEL(exit_unwind_notrace); \ + } \ + DTRACE_FUNCTION_ENTRY(); \ + next_instr = frame->instr_ptr; \ + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ + if (lltrace < 0) { \ + JUMP_TO_LABEL(exit_unwind); \ + } \ + frame->lltrace = lltrace; \ + assert(!_PyErr_Occurred(tstate)); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + _PyFrame_StackPointerInvalidate(frame); \ + { \ + int opcode; \ + DISPATCH(); \ + } \ + } +#else +#define START_FRAME() \ + LABEL(start_frame) { \ + int too_deep = _Py_EnterRecursivePy(tstate); \ + if (too_deep) { \ + JUMP_TO_LABEL(exit_unwind_notrace); \ + } \ + DTRACE_FUNCTION_ENTRY(); \ + next_instr = frame->instr_ptr; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + _PyFrame_StackPointerInvalidate(frame); \ + { \ + int opcode; \ + DISPATCH(); \ + } \ + } +#endif \ No newline at end of file diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6178dc70c1b80e7..d9339fa230ca0fc 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -13838,33 +13838,6 @@ JUMP_TO_LABEL(error); JUMP_TO_LABEL(error); } - LABEL(start_frame) - { - #ifdef Py_DEBUG - assert(frame->stackpointer_valid == 1); - #endif - int too_deep = _Py_EnterRecursivePy(tstate); - if (too_deep) { - JUMP_TO_LABEL(exit_unwind_notrace); - } - DTRACE_FUNCTION_ENTRY(); - next_instr = frame->instr_ptr; - #ifdef Py_DEBUG - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - if (lltrace < 0) { - JUMP_TO_LABEL(exit_unwind); - } - frame->lltrace = lltrace; - assert(!_PyErr_Occurred(tstate)); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - #if _Py_TAIL_CALL_INTERP - int opcode; - #endif - DISPATCH(); - } - #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) Py_GCC_ATTRIBUTE((unused)) #endif diff --git a/configure b/configure index 9159dd2fac6e7ba..ad5cfebd76cbd3a 100755 --- a/configure +++ b/configure @@ -31936,7 +31936,7 @@ fi printf "%s\n" "$reserve_frame_pointer_for_opcodes" >&6; } if test "$reserve_frame_pointer_for_opcodes" = yes; then - CFLAGS_CEVAL_TAILCALL="-mreserve-frame-pointer-reg" + CFLAGS_CEVAL_TAILCALL="-fomit-frame-pointer -momit-leaf-frame-pointer -mreserve-frame-pointer-reg" fi fi fi diff --git a/configure.ac b/configure.ac index 28e39571db19d44..359607f7aee8672 100644 --- a/configure.ac +++ b/configure.ac @@ -7466,7 +7466,7 @@ then AC_MSG_RESULT([$reserve_frame_pointer_for_opcodes]) if test "$reserve_frame_pointer_for_opcodes" = yes; then - CFLAGS_CEVAL_TAILCALL="-mreserve-frame-pointer-reg" + CFLAGS_CEVAL_TAILCALL="-fomit-frame-pointer -momit-leaf-frame-pointer -mreserve-frame-pointer-reg" fi fi fi From 71cf99df99d254619a67f97c23ac78a3f53c311e Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sat, 1 Aug 2026 10:39:47 +0100 Subject: [PATCH 04/14] Compatibility with clang 21 --- configure | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index ad5cfebd76cbd3a..5a7d5cdd38a4022 100755 --- a/configure +++ b/configure @@ -31936,7 +31936,7 @@ fi printf "%s\n" "$reserve_frame_pointer_for_opcodes" >&6; } if test "$reserve_frame_pointer_for_opcodes" = yes; then - CFLAGS_CEVAL_TAILCALL="-fomit-frame-pointer -momit-leaf-frame-pointer -mreserve-frame-pointer-reg" + CFLAGS_CEVAL_TAILCALL="-fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved" fi fi fi diff --git a/configure.ac b/configure.ac index 359607f7aee8672..8f2628d0cdee0aa 100644 --- a/configure.ac +++ b/configure.ac @@ -7466,7 +7466,7 @@ then AC_MSG_RESULT([$reserve_frame_pointer_for_opcodes]) if test "$reserve_frame_pointer_for_opcodes" = yes; then - CFLAGS_CEVAL_TAILCALL="-fomit-frame-pointer -momit-leaf-frame-pointer -mreserve-frame-pointer-reg" + CFLAGS_CEVAL_TAILCALL="-fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved" fi fi fi From 07e28319d26f49bb0b9a990ac6cd02a9289f4486 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 11 Oct 2026 21:36:17 +0100 Subject: [PATCH 05/14] Fix some stuff --- .../internal/pycore_ceval_opcode_targets.h | 2 +- configure | 64 ++++++++++++------- configure.ac | 31 ++++----- 3 files changed, 55 insertions(+), 42 deletions(-) diff --git a/Include/internal/pycore_ceval_opcode_targets.h b/Include/internal/pycore_ceval_opcode_targets.h index acf7f4bda30fd51..70b3ae543640506 100644 --- a/Include/internal/pycore_ceval_opcode_targets.h +++ b/Include/internal/pycore_ceval_opcode_targets.h @@ -61,7 +61,7 @@ extern "C" { } while (0) # endif # define LABEL(name) TARGET(name) -# include "Python/opcode_targets.h" +# include "opcode_targets.h" #endif #ifdef __cplusplus diff --git a/configure b/configure index 0999ffac50c5509..fd37ce0a4a5f28f 100755 --- a/configure +++ b/configure @@ -34415,44 +34415,62 @@ printf "%s\n" "#define _Py_TAIL_CALL_INTERP 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } + reserve_frame_pointer_for_opcodes=no if test -n "$frame_pointer_cflags" && test "x$with_frame_pointers" != xno; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we need to reserve the frame pointer in the opcode handlers" >&5 -printf %s "checking if we need to reserve the frame pointer in the opcode handlers... " >&6; } - if test "$cross_compiling" = yes + reserve_frame_pointer_for_opcodes=yes + fi + for flag in $CFLAGS; do + case $flag in #( + -fno-omit-frame-pointer|-fno-omit-leaf-frame-pointer|-mno-omit-leaf-frame-pointer) : + reserve_frame_pointer_for_opcodes=yes ;; #( + *) : + ;; +esac + done + if test "$reserve_frame_pointer_for_opcodes" = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved" >&5 +printf %s "checking whether C compiler accepts -fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved... " >&6; } +if test ${ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved+y} then : - reserve_frame_pointer_for_opcodes=undefined + printf %s "(cached) " >&6 else case e in #( - e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - int main(void) { - // clang-21 and newer support this CLI flag. - #if defined(__clang__) && (__clang_major__ >= 21) - return 1; - #else - return 0; - #endif - } +int +main (void) +{ + ; + return 0; +} _ACEOF -if ac_fn_c_try_run "$LINENO" +if ac_fn_c_try_compile "$LINENO" then : - reserve_frame_pointer_for_opcodes=no + ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved=yes else case e in #( - e) reserve_frame_pointer_for_opcodes=yes ;; + e) ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved=no ;; esac fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; esac fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved" >&6; } +if test "x$ax_cv_check_cflags__Werror__fomit_frame_pointer__momit_leaf_frame_pointer__Xclang__mframe_pointer_reserved" = xyes +then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $reserve_frame_pointer_for_opcodes" >&5 -printf "%s\n" "$reserve_frame_pointer_for_opcodes" >&6; } - - if test "$reserve_frame_pointer_for_opcodes" = yes; then CFLAGS_CEVAL_TAILCALL="-fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved" - fi + +else case e in #( + e) : ;; +esac +fi + fi fi if test "$withval" = no diff --git a/configure.ac b/configure.ac index fc04bd742affb9b..4c594b4982c3d5b 100644 --- a/configure.ac +++ b/configure.ac @@ -7709,26 +7709,21 @@ then AC_DEFINE([_Py_TAIL_CALL_INTERP], [1], [Define if you want to use tail-calling interpreters in CPython.]) AC_MSG_RESULT([yes]) + reserve_frame_pointer_for_opcodes=no if test -n "$frame_pointer_cflags" && test "x$with_frame_pointers" != xno; then - AC_MSG_CHECKING([if we need to reserve the frame pointer in the opcode handlers]) - AC_RUN_IFELSE([AC_LANG_SOURCE([[ - int main(void) { - // clang-21 and newer support this CLI flag. - #if defined(__clang__) && (__clang_major__ >= 21) - return 1; - #else - return 0; - #endif - } - ]])], - [reserve_frame_pointer_for_opcodes=no], - [reserve_frame_pointer_for_opcodes=yes], - [reserve_frame_pointer_for_opcodes=undefined]) - AC_MSG_RESULT([$reserve_frame_pointer_for_opcodes]) - - if test "$reserve_frame_pointer_for_opcodes" = yes; then + reserve_frame_pointer_for_opcodes=yes + fi + dnl Also recognize frame pointers requested explicitly in CFLAGS. + for flag in $CFLAGS; do + AS_CASE([$flag], + [-fno-omit-frame-pointer|-fno-omit-leaf-frame-pointer|-mno-omit-leaf-frame-pointer], + [reserve_frame_pointer_for_opcodes=yes]) + done + if test "$reserve_frame_pointer_for_opcodes" = yes; then + dnl Reserve the register without adding a frame to each tail-call handler. + AX_CHECK_COMPILE_FLAG([-fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved], [ CFLAGS_CEVAL_TAILCALL="-fomit-frame-pointer -momit-leaf-frame-pointer -Xclang -mframe-pointer=reserved" - fi + ], [], [-Werror]) fi fi if test "$withval" = no From ef6d729ff8b8d214d4dc28b90ce51af10603bf89 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 11 Oct 2026 21:45:35 +0100 Subject: [PATCH 06/14] Move more stuff to bytecodes.c --- Modules/_testinternalcapi/interpreter.c | 2 -- Modules/_testinternalcapi/test_cases.c.h | 27 ++++++++++++++++ Python/bytecodes.c | 28 ++++++++++++++++ Python/ceval.c | 6 ---- Python/ceval_macros.h | 41 ------------------------ Python/generated_cases.c.h | 27 ++++++++++++++++ 6 files changed, 82 insertions(+), 49 deletions(-) diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index 0660a0543b4854e..3d7dba5ec7e2e4d 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -35,7 +35,6 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) #include "test_targets.h" #undef _Py_CEVAL_OPCODE_TARGETS_DEFINE #include "test_cases.c.h" -START_FRAME(); #endif PyObject* _Py_HOT_FUNCTION @@ -145,7 +144,6 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) # endif #else goto start_frame; - START_FRAME(); #include "test_cases.c.h" #endif diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 1dbe23c4e18e068..3bdc16437e2bc61 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -13773,6 +13773,33 @@ JUMP_TO_LABEL(error); JUMP_TO_LABEL(error); } + LABEL(start_frame) + { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + JUMP_TO_LABEL(exit_unwind_notrace); + } + DTRACE_FUNCTION_ENTRY(); + next_instr = frame->instr_ptr; + #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; + assert(!_PyErr_Occurred(tstate)); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + #if _Py_TAIL_CALL_INTERP + int opcode; + #endif + DISPATCH(); + } + #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) Py_GCC_ATTRIBUTE((unused)) #endif diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fec78e891fd6552..18ed9b4cb9169b9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -6540,6 +6540,34 @@ dummy_func( goto error; } + spilled label(start_frame) { +#ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); +#endif + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + goto exit_unwind_notrace; + } + DTRACE_FUNCTION_ENTRY(); + next_instr = frame->instr_ptr; +#ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + frame->lltrace = lltrace; + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + * because it can clear it (directly or indirectly) and so the + * caller loses its exception. */ + assert(!_PyErr_Occurred(tstate)); +#endif + RELOAD_STACK(); +#if _Py_TAIL_CALL_INTERP + int opcode; +#endif + DISPATCH(); + } + inst(TRACE_RECORD, (--)) { #if _Py_TIER2 assert(IS_JIT_TRACING()); diff --git a/Python/ceval.c b/Python/ceval.c index 0eca01a7cf93579..e2001f94977c2e8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1223,11 +1223,6 @@ _PyEval_DTraceFunctionReturn(_PyInterpreterFrame *frame) } #endif -#if _Py_TAIL_CALL_INTERP -LABEL(exit_unwind_notrace); -START_FRAME(); -#endif - PyObject* _Py_HOT_FUNCTION DONT_SLP_VECTORIZE _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { @@ -1342,7 +1337,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int # endif #else goto start_frame; - START_FRAME(); # include "generated_cases.c.h" #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index fd5abf4f4474ba1..684d55ecebdb802 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -633,44 +633,3 @@ gen_try_set_executing(PyGenObject *gen) #define CALL_TP_ITERITEM_NO_ESCAPE(ITER, INDEX) \ Py_TYPE(ITER)->_tp_iteritem((ITER), (INDEX)) - -#if Py_DEBUG -#define START_FRAME() \ - LABEL(start_frame) { \ - assert(frame->stackpointer_valid == 1); \ - int too_deep = _Py_EnterRecursivePy(tstate); \ - if (too_deep) { \ - JUMP_TO_LABEL(exit_unwind_notrace); \ - } \ - DTRACE_FUNCTION_ENTRY(); \ - next_instr = frame->instr_ptr; \ - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ - if (lltrace < 0) { \ - JUMP_TO_LABEL(exit_unwind); \ - } \ - frame->lltrace = lltrace; \ - assert(!_PyErr_Occurred(tstate)); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - _PyFrame_StackPointerInvalidate(frame); \ - { \ - int opcode; \ - DISPATCH(); \ - } \ - } -#else -#define START_FRAME() \ - LABEL(start_frame) { \ - int too_deep = _Py_EnterRecursivePy(tstate); \ - if (too_deep) { \ - JUMP_TO_LABEL(exit_unwind_notrace); \ - } \ - DTRACE_FUNCTION_ENTRY(); \ - next_instr = frame->instr_ptr; \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - _PyFrame_StackPointerInvalidate(frame); \ - { \ - int opcode; \ - DISPATCH(); \ - } \ - } -#endif \ No newline at end of file diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 770ec485b5cac4c..dd0ce41e4b06b4a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -13770,6 +13770,33 @@ JUMP_TO_LABEL(error); JUMP_TO_LABEL(error); } + LABEL(start_frame) + { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + JUMP_TO_LABEL(exit_unwind_notrace); + } + DTRACE_FUNCTION_ENTRY(); + next_instr = frame->instr_ptr; + #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; + assert(!_PyErr_Occurred(tstate)); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + #if _Py_TAIL_CALL_INTERP + int opcode; + #endif + DISPATCH(); + } + #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) Py_GCC_ATTRIBUTE((unused)) #endif From 83690e18f2ac715e080c60ef4b2dfa99d9663427 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 11 Oct 2026 21:48:01 +0100 Subject: [PATCH 07/14] Revert unrelated formatting diff --- Python/bytecodes.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 18ed9b4cb9169b9..31eaeab0d67841f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -6541,26 +6541,26 @@ dummy_func( } spilled label(start_frame) { -#ifdef Py_DEBUG + #ifdef Py_DEBUG assert(frame->stackpointer_valid == 1); -#endif + #endif int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { goto exit_unwind_notrace; } DTRACE_FUNCTION_ENTRY(); next_instr = frame->instr_ptr; -#ifdef Py_DEBUG + #ifdef Py_DEBUG int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); if (lltrace < 0) { - goto exit_unwind; + JUMP_TO_LABEL(exit_unwind); } frame->lltrace = lltrace; /* _PyEval_EvalFrameDefault() must not be called with an exception set, - * because it can clear it (directly or indirectly) and so the - * caller loses its exception. */ + because it can clear it (directly or indirectly) and so the + caller loses its exception */ assert(!_PyErr_Occurred(tstate)); -#endif + #endif RELOAD_STACK(); #if _Py_TAIL_CALL_INTERP int opcode; @@ -6650,6 +6650,7 @@ dummy_func( exit_unwind_notrace: handle_eval_breaker: resume_frame: + start_frame: unbound_local_error: ; } From f5eec96139083ab57c6ec046d8bd67ddcf905951 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 11 Oct 2026 21:49:58 +0100 Subject: [PATCH 08/14] fix include --- Modules/_testinternalcapi/interpreter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index 3d7dba5ec7e2e4d..ce08aaa681a68d4 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -8,7 +8,7 @@ #include "../../Python/ceval.h" #include "../../Python/ceval_macros.h" -#include "pycore_ceval_opcode_targets.h" +#include "../../Include/internal/pycore_ceval_opcode_targets.h" #undef IS_PEP523_HOOKED #define IS_PEP523_HOOKED(tstate) (tstate->interp->eval_frame != NULL && !tstate->interp->eval_frame_allow_specialization) From c2de28ac1ecdfc46e5a9b8eb89f4892737d3be57 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 11 Oct 2026 21:55:54 +0100 Subject: [PATCH 09/14] reduce diff --- .../internal/pycore_ceval_opcode_targets.h | 70 ------------------- Makefile.pre.in | 4 +- Modules/_testinternalcapi/interpreter.c | 1 - Python/ceval.c | 5 +- Python/ceval_macros.h | 53 ++++++++++++++ Python/ceval_tailcall.c | 2 +- 6 files changed, 59 insertions(+), 76 deletions(-) delete mode 100644 Include/internal/pycore_ceval_opcode_targets.h diff --git a/Include/internal/pycore_ceval_opcode_targets.h b/Include/internal/pycore_ceval_opcode_targets.h deleted file mode 100644 index 70b3ae543640506..000000000000000 --- a/Include/internal/pycore_ceval_opcode_targets.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef Py_INTERNAL_CEVAL_OPCODE_TARGETS_H -#define Py_INTERNAL_CEVAL_OPCODE_TARGETS_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#ifdef Py_STATS -# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, int lastopcode -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, lastopcode -#else -# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg -# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg -#endif - -#if _Py_TAIL_CALL_INTERP -# if defined(__clang__) || defined(__GNUC__) -# if !_Py__has_attribute(preserve_none) || !_Py__has_attribute(musttail) -# error "This compiler does not have support for efficient tail calling." -# endif -# elif defined(_MSC_VER) && (_MSC_VER < 1950) -# error "You need at least VS 2026 / PlatformToolset v145 for tail calling." -# endif -# if defined(_MSC_VER) && !defined(__clang__) -# define Py_MUSTTAIL [[msvc::musttail]] -# define Py_PRESERVE_NONE_CC __preserve_none -# else -# define Py_MUSTTAIL __attribute__((musttail)) -# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) -# endif -# define DISPATCH_TABLE_VAR instruction_funcptr_table -# define DISPATCH_TABLE instruction_funcptr_handler_table -# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table -# define TARGET(op) Py_NO_INLINE PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_##op(TAIL_CALL_PARAMS) - typedef PyObject *(Py_PRESERVE_NONE_CC *py_tail_call_funcptr)(TAIL_CALL_PARAMS); - -# define DISPATCH_GOTO() \ - do { \ - Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ - } while (0) -# define DISPATCH_GOTO_NON_TRACING() \ - do { \ - Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ - } while (0) -# define JUMP_TO_LABEL(name) \ - do { \ - Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ - } while (0) -# ifdef Py_STATS -# define JUMP_TO_PREDICTED(name) \ - do { \ - Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, lastopcode); \ - } while (0) -# else -# define JUMP_TO_PREDICTED(name) \ - do { \ - Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg); \ - } while (0) -# endif -# define LABEL(name) TARGET(name) -# include "opcode_targets.h" -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERNAL_CEVAL_OPCODE_TARGETS_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 17600ee5a209903..b79ec18c18dcb2a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1318,7 +1318,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_cell.h \ $(srcdir)/Include/internal/pycore_ceval.h \ $(srcdir)/Include/internal/pycore_ceval_state.h \ - $(srcdir)/Include/internal/pycore_ceval_opcode_targets.h \ $(srcdir)/Include/internal/pycore_code.h \ $(srcdir)/Include/internal/pycore_codecs.h \ $(srcdir)/Include/internal/pycore_compile.h \ @@ -2203,7 +2202,6 @@ Python/ceval.o: \ $(srcdir)/Python/record_functions.c.h Python/ceval_tailcall.o: \ - $(srcdir)/Include/internal/pycore_ceval_opcode_targets.h \ $(srcdir)/Python/ceval.h \ $(srcdir)/Python/ceval_macros.h \ $(srcdir)/Python/generated_cases.c.h \ @@ -3479,7 +3477,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo. MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h -MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Include/internal/pycore_ceval_opcode_targets.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h +MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Python/ceval_macros.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index ce08aaa681a68d4..56fea28f3ec609d 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -8,7 +8,6 @@ #include "../../Python/ceval.h" #include "../../Python/ceval_macros.h" -#include "../../Include/internal/pycore_ceval_opcode_targets.h" #undef IS_PEP523_HOOKED #define IS_PEP523_HOOKED(tstate) (tstate->interp->eval_frame != NULL && !tstate->interp->eval_frame_allow_specialization) diff --git a/Python/ceval.c b/Python/ceval.c index e2001f94977c2e8..69ecc98dfecd5c2 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2,7 +2,6 @@ #include "ceval.h" #include "pycore_long.h" -#include "pycore_ceval_opcode_targets.h" int Py_GetRecursionLimit(void) @@ -1110,6 +1109,10 @@ _PyEval_StopTracingAndJit(PyThreadState *tstate, _PyInterpreterFrame *frame) /* This setting is reversed below following _PyEval_EvalFrameDefault */ #endif +#if _Py_TAIL_CALL_INTERP +#include "opcode_targets.h" +#endif + _PyStackRef _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *index_or_null, int yield_from) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 684d55ecebdb802..721d66fd502bd45 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -71,7 +71,60 @@ #define INSTRUCTION_STATS(op) ((void)0) #endif +#ifdef Py_STATS +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, int lastopcode +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, lastopcode +#else +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg +#endif + #if _Py_TAIL_CALL_INTERP +# if defined(__clang__) || defined(__GNUC__) +# if !_Py__has_attribute(preserve_none) || !_Py__has_attribute(musttail) +# error "This compiler does not have support for efficient tail calling." +# endif +# elif defined(_MSC_VER) && (_MSC_VER < 1950) +# error "You need at least VS 2026 / PlatformToolset v145 for tail calling." +# endif +# if defined(_MSC_VER) && !defined(__clang__) +# define Py_MUSTTAIL [[msvc::musttail]] +# define Py_PRESERVE_NONE_CC __preserve_none +# else +# define Py_MUSTTAIL __attribute__((musttail)) +# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) +# endif + typedef PyObject *(Py_PRESERVE_NONE_CC *py_tail_call_funcptr)(TAIL_CALL_PARAMS); + +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table +# define TARGET(op) Py_NO_INLINE PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_##op(TAIL_CALL_PARAMS) + +# define DISPATCH_GOTO() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define JUMP_TO_LABEL(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ + } while (0) +# ifdef Py_STATS +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, lastopcode); \ + } while (0) +# else +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg); \ + } while (0) +# endif +# define LABEL(name) TARGET(name) #elif USE_COMPUTED_GOTOS # define DISPATCH_TABLE_VAR opcode_targets # define DISPATCH_TABLE opcode_targets_table diff --git a/Python/ceval_tailcall.c b/Python/ceval_tailcall.c index 566ee1ea8009437..6aacab81eef318a 100644 --- a/Python/ceval_tailcall.c +++ b/Python/ceval_tailcall.c @@ -5,7 +5,7 @@ #if _Py_TAIL_CALL_INTERP #define _Py_CEVAL_OPCODE_TARGETS_DEFINE -#include "pycore_ceval_opcode_targets.h" +#include "opcode_targets.h" #undef _Py_CEVAL_OPCODE_TARGETS_DEFINE #include "generated_cases.c.h" #endif From ec97045cf59f30597d7fb51d00fe667cc6b3c900 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 11 Oct 2026 22:17:44 +0100 Subject: [PATCH 10/14] Move things to ceval.c --- Lib/test/test_generated_cases.py | 40 +++++++++++++++ Makefile.pre.in | 2 - Modules/_testinternalcapi/interpreter.c | 3 ++ Modules/_testinternalcapi/test_cases.c.h | 59 ++++++++++++----------- Modules/_testinternalcapi/test_targets.h | 4 +- Python/ceval.c | 4 ++ Python/generated_cases.c.h | 59 ++++++++++++----------- Python/opcode_targets.h | 4 +- Tools/cases_generator/target_generator.py | 2 +- Tools/cases_generator/tier1_generator.py | 9 +++- 10 files changed, 124 insertions(+), 62 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 8ce5370e922e069..248abd6e038c856 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -124,6 +124,7 @@ def run_cases_test(self, input: str, expected: str): actual = instructions.strip() + "\n\n " + labels.strip() self.assertEqual(actual.strip(), expected.strip()) + return lines def test_inst_no_args(self): input = """ @@ -1791,6 +1792,45 @@ def test_spilled_label(self): """ self.run_cases_test(input, output) + def test_start_frame_separate_from_handlers(self): + input = """ + spilled label(start_frame) { + RELOAD_STACK(); + goto handler; + } + + label(handler) { + SAVE_STACK(); + goto start_frame; + } + """ + output = """ + LABEL(handler) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + JUMP_TO_LABEL(start_frame); + } + """ + generated = self.run_cases_test(input, output) + handlers, start_frame = generated.split( + "#if !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME)\n" + ) + self.assertNotIn("\n LABEL(start_frame)\n", handlers) + self.assertIn("#ifndef _Py_CEVAL_START_FRAME", handlers) + expected = """ + LABEL(start_frame) + { + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(handler); + } + +#endif /* !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) */ +#undef TIER_ONE + """ + self.assertEqual(start_frame.strip(), expected.strip()) + def test_incorrect_spills(self): input1 = """ diff --git a/Makefile.pre.in b/Makefile.pre.in index b79ec18c18dcb2a..4a2526b5e8b8149 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2208,8 +2208,6 @@ Python/ceval_tailcall.o: \ $(srcdir)/Python/opcode_targets.h \ $(srcdir)/Python/record_functions.c.h -Modules/_testinternalcapi/interpreter.o: PY_STDMODULE_CFLAGS += $(CFLAGS_CEVAL_TAILCALL) - Python/flowgraph.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index 56fea28f3ec609d..a1d14068dc3d8e1 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -34,6 +34,9 @@ stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) #include "test_targets.h" #undef _Py_CEVAL_OPCODE_TARGETS_DEFINE #include "test_cases.c.h" +#define _Py_CEVAL_START_FRAME +#include "test_cases.c.h" +#undef _Py_CEVAL_START_FRAME #endif PyObject* _Py_HOT_FUNCTION diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 3bdc16437e2bc61..7e92e06e0e197d2 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -7,6 +7,7 @@ #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 +#ifndef _Py_CEVAL_START_FRAME #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS @@ -13773,33 +13774,6 @@ JUMP_TO_LABEL(error); JUMP_TO_LABEL(error); } - LABEL(start_frame) - { - #ifdef Py_DEBUG - assert(frame->stackpointer_valid == 1); - #endif - int too_deep = _Py_EnterRecursivePy(tstate); - if (too_deep) { - JUMP_TO_LABEL(exit_unwind_notrace); - } - DTRACE_FUNCTION_ENTRY(); - next_instr = frame->instr_ptr; - #ifdef Py_DEBUG - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - if (lltrace < 0) { - JUMP_TO_LABEL(exit_unwind); - } - frame->lltrace = lltrace; - assert(!_PyErr_Occurred(tstate)); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - #if _Py_TAIL_CALL_INTERP - int opcode; - #endif - DISPATCH(); - } - #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) Py_GCC_ATTRIBUTE((unused)) #endif @@ -13827,4 +13801,35 @@ JUMP_TO_LABEL(error); } /* END LABELS */ +#endif /* _Py_CEVAL_START_FRAME */ +#if !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) + + LABEL(start_frame) + { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + JUMP_TO_LABEL(exit_unwind_notrace); + } + DTRACE_FUNCTION_ENTRY(); + next_instr = frame->instr_ptr; + #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; + assert(!_PyErr_Occurred(tstate)); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + #if _Py_TAIL_CALL_INTERP + int opcode; + #endif + DISPATCH(); + } + +#endif /* !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) */ #undef TIER_ONE diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 9518c53569fb5ea..f42531563f85768 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -524,13 +524,13 @@ extern py_tail_call_funcptr instruction_funcptr_tracing_table[256]; extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); #ifdef _Py_CEVAL_OPCODE_TARGETS_DEFINE extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); -extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); diff --git a/Python/ceval.c b/Python/ceval.c index 69ecc98dfecd5c2..7209659c063d242 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1111,6 +1111,10 @@ _PyEval_StopTracingAndJit(PyThreadState *tstate, _PyInterpreterFrame *frame) #if _Py_TAIL_CALL_INTERP #include "opcode_targets.h" +/* Give start_frame its own frame pointer, independently of the handlers. */ +#define _Py_CEVAL_START_FRAME +#include "generated_cases.c.h" +#undef _Py_CEVAL_START_FRAME #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dd0ce41e4b06b4a..94aee7d8254a9bd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7,6 +7,7 @@ #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 +#ifndef _Py_CEVAL_START_FRAME #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS @@ -13770,33 +13771,6 @@ JUMP_TO_LABEL(error); JUMP_TO_LABEL(error); } - LABEL(start_frame) - { - #ifdef Py_DEBUG - assert(frame->stackpointer_valid == 1); - #endif - int too_deep = _Py_EnterRecursivePy(tstate); - if (too_deep) { - JUMP_TO_LABEL(exit_unwind_notrace); - } - DTRACE_FUNCTION_ENTRY(); - next_instr = frame->instr_ptr; - #ifdef Py_DEBUG - int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); - if (lltrace < 0) { - JUMP_TO_LABEL(exit_unwind); - } - frame->lltrace = lltrace; - assert(!_PyErr_Occurred(tstate)); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - #if _Py_TAIL_CALL_INTERP - int opcode; - #endif - DISPATCH(); - } - #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) Py_GCC_ATTRIBUTE((unused)) #endif @@ -13824,4 +13798,35 @@ JUMP_TO_LABEL(error); } /* END LABELS */ +#endif /* _Py_CEVAL_START_FRAME */ +#if !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) + + LABEL(start_frame) + { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + JUMP_TO_LABEL(exit_unwind_notrace); + } + DTRACE_FUNCTION_ENTRY(); + next_instr = frame->instr_ptr; + #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; + assert(!_PyErr_Occurred(tstate)); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + #if _Py_TAIL_CALL_INTERP + int opcode; + #endif + DISPATCH(); + } + +#endif /* !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) */ #undef TIER_ONE diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 9518c53569fb5ea..f42531563f85768 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -524,13 +524,13 @@ extern py_tail_call_funcptr instruction_funcptr_tracing_table[256]; extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); #ifdef _Py_CEVAL_OPCODE_TARGETS_DEFINE extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); -extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); -extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); extern PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index aa3e6a46558f917..803229499391da4 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -51,7 +51,7 @@ def function_proto(name: str) -> str: return f"extern {function_signature(name)}" def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: - shared_labels = ("error", "start_frame") + shared_labels = ("error", "start_frame", "exit_unwind", "exit_unwind_notrace") out.emit("extern py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") out.emit("\n") out.emit("extern py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 0d477cb83c5d436..ee2000502e5c46a 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -149,6 +149,7 @@ def generate_tier1( #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 +#ifndef _Py_CEVAL_START_FRAME """) outfile.write(f""" #if !_Py_TAIL_CALL_INTERP @@ -187,16 +188,22 @@ def generate_tier1( emitter = Emitter(out, analysis.labels) generate_tier1_labels(analysis, emitter) outfile.write(f"{LABEL_END_MARKER}\n") + outfile.write("#endif /* _Py_CEVAL_START_FRAME */\n") + outfile.write("#if !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME)\n") + generate_tier1_labels(analysis, emitter, start_frame=True) + outfile.write("#endif /* !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) */\n") outfile.write(FOOTER) def generate_tier1_labels( - analysis: Analysis, emitter: Emitter + analysis: Analysis, emitter: Emitter, *, start_frame: bool = False ) -> None: emitter.emit("\n") # Emit tail-callable labels as function defintions for name, label in analysis.labels.items(): + if (name == "start_frame") != start_frame: + continue if name == 'stop_tracing': emitter.emit("#if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2)\n") emitter.emit("Py_GCC_ATTRIBUTE((unused))\n") From 67f5b86777e9c701ec034a5167e2211862d4c63a Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 11 Oct 2026 22:48:59 +0100 Subject: [PATCH 11/14] simplify code --- Tools/cases_generator/tier1_generator.py | 45 ++++++++++++++---------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index ee2000502e5c46a..27792b4a945e05c 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -8,6 +8,7 @@ from analyzer import ( Analysis, Instruction, + Label, Uop, Part, analyze_files, @@ -190,31 +191,39 @@ def generate_tier1( outfile.write(f"{LABEL_END_MARKER}\n") outfile.write("#endif /* _Py_CEVAL_START_FRAME */\n") outfile.write("#if !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME)\n") - generate_tier1_labels(analysis, emitter, start_frame=True) + generate_tier1_start_frame(analysis, emitter) outfile.write("#endif /* !_Py_TAIL_CALL_INTERP || defined(_Py_CEVAL_START_FRAME) */\n") outfile.write(FOOTER) -def generate_tier1_labels( - analysis: Analysis, emitter: Emitter, *, start_frame: bool = False -) -> None: +def generate_tier1_label(label: Label, emitter: Emitter) -> None: + if label.name == 'stop_tracing': + emitter.emit("#if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2)\n") + emitter.emit("Py_GCC_ATTRIBUTE((unused))\n") + emitter.emit("#endif\n") + emitter.emit(f"LABEL({label.name})\n") + storage = Storage(Stack(), [], [], 0, False) + if label.spilled: + storage.spilled = 1 + storage.stack.physical_sp = None + emitter.emit_tokens(label, storage, None) + emitter.emit("\n\n") + + +def generate_tier1_labels(analysis: Analysis, emitter: Emitter) -> None: emitter.emit("\n") # Emit tail-callable labels as function defintions - for name, label in analysis.labels.items(): - if (name == "start_frame") != start_frame: - continue - if name == 'stop_tracing': - emitter.emit("#if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2)\n") - emitter.emit("Py_GCC_ATTRIBUTE((unused))\n") - emitter.emit("#endif\n") - emitter.emit(f"LABEL({name})\n") - storage = Storage(Stack(), [], [], 0, False) - if label.spilled: - storage.spilled = 1 - storage.stack.physical_sp = None - emitter.emit_tokens(label, storage, None) - emitter.emit("\n\n") + for label in analysis.labels.values(): + if label.name != "start_frame": + generate_tier1_label(label, emitter) + + +def generate_tier1_start_frame(analysis: Analysis, emitter: Emitter) -> None: + emitter.emit("\n") + label = analysis.labels.get("start_frame") + if label is not None: + generate_tier1_label(label, emitter) def get_popped(inst: Instruction, analysis: Analysis) -> str: stack = get_stack_effect(inst) From f23dc2443a0f25b3a415afd7175d66f0097898cf Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:57:50 +0000 Subject: [PATCH 12/14] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-10-11-21-57-44.gh-issue-154124.ZErMAp.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-21-57-44.gh-issue-154124.ZErMAp.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-21-57-44.gh-issue-154124.ZErMAp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-21-57-44.gh-issue-154124.ZErMAp.rst new file mode 100644 index 000000000000000..d5a651c77a5d793 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-21-57-44.gh-issue-154124.ZErMAp.rst @@ -0,0 +1 @@ +Reserve frame pointer on tail calling builds using newer Clang. This recovers most of the performance hit from frame pointers on tail calling builds using recent versions of Clang From 9fbd53e7dbd237631dee5a2d0e6d5aca77e36c99 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 11 Oct 2026 23:10:59 +0100 Subject: [PATCH 13/14] fix GCC --- Python/ceval_tailcall.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/ceval_tailcall.c b/Python/ceval_tailcall.c index 6aacab81eef318a..66debb96b3c4575 100644 --- a/Python/ceval_tailcall.c +++ b/Python/ceval_tailcall.c @@ -1,9 +1,11 @@ +#include "pyconfig.h" // _Py_TAIL_CALL_INTERP + +#if _Py_TAIL_CALL_INTERP #include "ceval.h" #include "pycore_long.h" #include "ceval_macros.h" -#if _Py_TAIL_CALL_INTERP #define _Py_CEVAL_OPCODE_TARGETS_DEFINE #include "opcode_targets.h" #undef _Py_CEVAL_OPCODE_TARGETS_DEFINE From 7eb8bd583da324586a3ff0750148d784e2ee07ca Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 11 Oct 2026 23:18:17 +0100 Subject: [PATCH 14/14] fix for msvc tailcall --- PCbuild/pythoncore.vcxproj | 3 +++ PCbuild/pythoncore.vcxproj.filters | 3 +++ 2 files changed, 6 insertions(+) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c2d8f64a72b1191..53ec585ac436086 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -626,6 +626,9 @@ /std:clatest %(AdditionalOptions) + + /std:clatest %(AdditionalOptions) + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 2536fc971700deb..8da1b797e3a0009 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1421,6 +1421,9 @@ Python + + Python + Python