From a6fffba4c00cddbaf002d78a2307633a09e58bc7 Mon Sep 17 00:00:00 2001 From: yotatameguchi-boop <309382109+yotatameguchi-boop@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:19:01 +0900 Subject: [PATCH] Implement experimental PEP 671 late-bound argument defaults --- Doc/howto/index.rst | 1 + Doc/howto/late-bound-defaults.rst | 79 ++++ Grammar/python.gram | 14 +- Include/cpython/code.h | 2 + Include/cpython/funcobject.h | 2 + Include/internal/pycore_ast.h | 7 +- Include/internal/pycore_ast_state.h | 1 + .../pycore_global_objects_fini_generated.h | 2 + Include/internal/pycore_global_strings.h | 2 + Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 20 +- Include/internal/pycore_opcode_utils.h | 2 + .../internal/pycore_runtime_init_generated.h | 2 + Include/internal/pycore_symtable.h | 1 + .../internal/pycore_unicodeobject_generated.h | 2 + Include/internal/pycore_uop_ids.h | 8 +- Include/internal/pycore_uop_metadata.h | 34 ++ Include/opcode_ids.h | 36 +- Lib/_ast_unparse.py | 4 +- Lib/_opcode_metadata.py | 36 +- Lib/dis.py | 3 +- Lib/inspect.py | 53 ++- Lib/test/test_ast/snippets.py | 36 +- Lib/test/test_late_defaults.py | 406 ++++++++++++++++++ Lib/test/test_sys.py | 2 +- .../2026-10-15-17-00-00.pep-671.rst | 3 + Modules/_testinternalcapi/test_cases.c.h | 39 ++ Modules/_testinternalcapi/test_targets.h | 18 +- Objects/funcobject.c | 107 +++++ Parser/Python.asdl | 2 +- Parser/action_helpers.c | 14 +- Parser/parser.c | 322 +++++++++++++- Parser/pegen.h | 1 + Python/Python-ast.c | 78 +++- Python/ast.c | 23 + Python/ast_unparse.c | 7 +- Python/bytecodes.c | 12 + Python/ceval.c | 54 ++- Python/codegen.c | 131 +++++- Python/compile.c | 6 +- Python/executor_cases.c.h | 70 +++ Python/flowgraph.c | 1 + Python/generated_cases.c.h | 39 ++ Python/opcode_targets.h | 18 +- Python/optimizer_cases.c.h | 20 + Python/symtable.c | 93 +++- 46 files changed, 1654 insertions(+), 162 deletions(-) create mode 100644 Doc/howto/late-bound-defaults.rst create mode 100644 Lib/test/test_late_defaults.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2026-10-15-17-00-00.pep-671.rst diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index 57e2d6e0752447f..65bd6c1be2526c9 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -32,6 +32,7 @@ Python Library Reference. instrumentation.rst perf_profiling.rst annotations.rst + late-bound-defaults.rst isolating-extensions.rst timerfd.rst mro.rst diff --git a/Doc/howto/late-bound-defaults.rst b/Doc/howto/late-bound-defaults.rst new file mode 100644 index 000000000000000..3a3fa8fee82aa1d --- /dev/null +++ b/Doc/howto/late-bound-defaults.rst @@ -0,0 +1,79 @@ +.. _late-bound-defaults-howto: + +Experimental late-bound argument defaults +======================================== + +.. warning:: + + This branch implements draft :pep:`671` for experimentation. The PEP has + not been accepted, and this syntax is not available in released Python. + +A default written with ``=`` is evaluated when a function is defined. A default +written with ``=>`` is evaluated in the function's local scope when the argument +is omitted:: + + def add_item(item, target=>[]): + target.append(item) + return target + + assert add_item(1) == [1] + assert add_item(2) == [2] + +An explicitly supplied argument, including ``None`` or ``Ellipsis``, suppresses +evaluation of its default. Late-bound defaults are evaluated from left to right +and can refer to earlier arguments:: + + def window(items, start=0, stop=>len(items)): + return items[start:stop] + +Positional-only and keyword-only parameters, annotated parameters, methods, +generic functions, and lambdas support the same syntax. All supplied arguments +and early-bound defaults are bound before late-bound defaults are evaluated. +An unbound reference raises :exc:`UnboundLocalError`:: + + def invalid(value=>value): + return value + + # invalid() raises UnboundLocalError; invalid(3) returns 3. + +Coroutine and generator prologues run when execution starts, as does their +function body. An argument error is reported before any default is evaluated. + +Introspection +------------- + +``function.__defaults__`` and ``function.__kwdefaults__`` contain ``Ellipsis`` +placeholders for late-bound parameters. ``function.__defaults_extra__`` holds a +parallel tuple of normalized expression strings (or ``None`` for early-bound +entries), and ``function.__kwdefaults_extra__`` holds expression strings keyed +by keyword-only parameter name. Both metadata attributes are ``None`` when no +corresponding late-bound defaults exist. These attributes can be replaced or +deleted using the same tuple/dict restrictions as the ordinary defaults. + +``inspect.signature()`` displays the ``=>`` expression. Its parameters expose +the source string through ``Parameter.late_default``. ``getfullargspec()`` keeps +its existing seven-field result; its defaults contain the same placeholders as +the function attributes. Signature binding does not execute expressions. + +``ast.arg.late_default`` is an optional string containing the normalized source +expression. The actual expression remains in ``arguments.defaults`` or +``arguments.kw_defaults``. This preserves the existing defaults fields and +allows :func:`ast.unparse` to reconstruct the new syntax. + +Implementation notes +-------------------- + +The call binder leaves omitted late-bound arguments unbound. Function prologues +use ``QUERY_FAST`` and ``QUERY_DEREF`` to determine which expressions to evaluate. +``QUERY_DEREF`` supports parameters captured by closures. Expression descriptions +are captured before AST constant folding. Code objects using late-bound defaults +carry the experimental ``CO_LATE_DEFAULTS`` flag. + +Existing call specializations are disabled for these functions because they +assume all arguments are bound before entering a frame. The compiler also treats +all parameters as potentially unbound during local-variable analysis. More +precise analysis and specialized call paths are future performance work. + +Two metadata pointers are added to each function object. This prototype changes +the internal function layout and bytecode magic number; extensions must be +rebuilt for it. It does not add public C API entry points. diff --git a/Grammar/python.gram b/Grammar/python.gram index f2c9993aa4f80a7..5486943fd8ec5fb 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -369,13 +369,17 @@ param_no_default_star_annotation[arg_ty]: | a=param_star_annotation ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) } | a=param_star_annotation tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) } param_with_default[NameDefaultPair*]: + | a=param '=' '>' c=expression ',' tc=TYPE_COMMENT? { _PyPegen_late_default_pair(p, a, c, tc) } + | a=param '=' '>' c=expression tc=TYPE_COMMENT? &')' { _PyPegen_late_default_pair(p, a, c, tc) } | a=param c=default ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) } | a=param c=default tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) } param_maybe_default[NameDefaultPair*]: + | a=param '=' '>' c=expression ',' tc=TYPE_COMMENT? { _PyPegen_late_default_pair(p, a, c, tc) } + | a=param '=' '>' c=expression tc=TYPE_COMMENT? &')' { _PyPegen_late_default_pair(p, a, c, tc) } | a=param c=default? ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) } | a=param c=default? tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) } -param[arg_ty]: a=NAME b=annotation? { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) } -param_star_annotation[arg_ty]: a=NAME b=star_annotation { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) } +param[arg_ty]: a=NAME b=annotation? { _PyAST_arg(a->v.Name.id, b, NULL, NULL, EXTRA) } +param_star_annotation[arg_ty]: a=NAME b=star_annotation { _PyAST_arg(a->v.Name.id, b, NULL, NULL, EXTRA) } annotation[expr_ty]: ':' a=expression { a } star_annotation[expr_ty]: ':' a=star_expression { a } default[expr_ty]: '=' a=expression { a } | invalid_default @@ -941,12 +945,16 @@ lambda_param_no_default[arg_ty]: | a=lambda_param ',' { a } | a=lambda_param &':' { a } lambda_param_with_default[NameDefaultPair*]: + | a=lambda_param '=' '>' c=expression ',' { _PyPegen_late_default_pair(p, a, c, NULL) } + | a=lambda_param '=' '>' c=expression &':' { _PyPegen_late_default_pair(p, a, c, NULL) } | a=lambda_param c=default ',' { _PyPegen_name_default_pair(p, a, c, NULL) } | a=lambda_param c=default &':' { _PyPegen_name_default_pair(p, a, c, NULL) } lambda_param_maybe_default[NameDefaultPair*]: + | a=lambda_param '=' '>' c=expression ',' { _PyPegen_late_default_pair(p, a, c, NULL) } + | a=lambda_param '=' '>' c=expression &':' { _PyPegen_late_default_pair(p, a, c, NULL) } | a=lambda_param c=default? ',' { _PyPegen_name_default_pair(p, a, c, NULL) } | a=lambda_param c=default? &':' { _PyPegen_name_default_pair(p, a, c, NULL) } -lambda_param[arg_ty]: a=NAME { _PyAST_arg(a->v.Name.id, NULL, NULL, EXTRA) } +lambda_param[arg_ty]: a=NAME { _PyAST_arg(a->v.Name.id, NULL, NULL, NULL, EXTRA) } # LITERALS # ======== diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 1be47e42ed62bfb..c647f75929923e6 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -131,6 +131,8 @@ struct PyCodeObject _PyCode_DEF(1); #define CO_COROUTINE 0x0080 #define CO_ITERABLE_COROUTINE 0x0100 #define CO_ASYNC_GENERATOR 0x0200 +/* Experimental PEP 671 call-time default evaluation. */ +#define CO_LATE_DEFAULTS 0x0400 /* bpo-39562: These constant values are changed in Python 3.9 to prevent collision with compiler flags. CO_FUTURE_ and PyCF_ diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 9e1599a76485646..3f6defdd5409dee 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -36,6 +36,8 @@ typedef struct { typedef struct { PyObject_HEAD _Py_COMMON_FIELDS(func_) + PyObject *func_defaults_extra; /* NULL or tuple of expression strings */ + PyObject *func_kwdefaults_extra; /* NULL or dict of expression strings */ PyObject *func_doc; /* The __doc__ attribute, can be anything */ PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ PyObject *func_weakreflist; /* List of weak references */ diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index b47398669bbe513..6b32ff6c998a2b5 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -565,6 +565,7 @@ struct _arg { identifier arg; expr_ty annotation; string type_comment; + string late_default; int lineno; int col_offset; int end_lineno; @@ -878,9 +879,9 @@ arguments_ty _PyAST_arguments(asdl_arg_seq * posonlyargs, asdl_arg_seq * args, arg_ty vararg, asdl_arg_seq * kwonlyargs, asdl_expr_seq * kw_defaults, arg_ty kwarg, asdl_expr_seq * defaults, PyArena *arena); -arg_ty _PyAST_arg(identifier arg, expr_ty annotation, string type_comment, int - lineno, int col_offset, int end_lineno, int end_col_offset, - PyArena *arena); +arg_ty _PyAST_arg(identifier arg, expr_ty annotation, string type_comment, + string late_default, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); keyword_ty _PyAST_keyword(identifier arg, expr_ty value, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index 32c12fb5875e8ea..9d56ff4deb849d7 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -219,6 +219,7 @@ struct ast_state { PyObject *kwd_attrs; PyObject *kwd_patterns; PyObject *kwonlyargs; + PyObject *late_default; PyObject *left; PyObject *level; PyObject *lineno; diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 0e2fab84aec0ae6..40f22d20dfd8b81 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -34,6 +34,7 @@ _PyStaticObjects_CheckAll(PyInterpreterState *interp) { _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(dbl_open_br), "{{", 2); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(dbl_percent), "%%", 2); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(defaults), ".defaults", 9); + _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(defaults_extra), ".defaults_extra", 15); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(dot_locals), ".", 9); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(empty), "", 0); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(format), ".format", 7); @@ -41,6 +42,7 @@ _PyStaticObjects_CheckAll(PyInterpreterState *interp) { _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(generic_base), ".generic_base", 13); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(json_decoder), "json.decoder", 12); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(kwdefaults), ".kwdefaults", 11); + _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(kwdefaults_extra), ".kwdefaults_extra", 17); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(list_err), "list index out of range", 23); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(native), "", 8); _PyStaticObject_CheckUnicodeSingleton((PyObject *)&_Py_STR(str_replace_inf), "1e309", 5); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index cbfa80f76b64cd1..a171f35e3aa73fd 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -43,6 +43,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(dbl_open_br, "{{") STRUCT_FOR_STR(dbl_percent, "%%") STRUCT_FOR_STR(defaults, ".defaults") + STRUCT_FOR_STR(defaults_extra, ".defaults_extra") STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") STRUCT_FOR_STR(format, ".format") @@ -50,6 +51,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") + STRUCT_FOR_STR(kwdefaults_extra, ".kwdefaults_extra") STRUCT_FOR_STR(list_err, "list index out of range") STRUCT_FOR_STR(native, "") STRUCT_FOR_STR(str_replace_inf, "1e309") diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 15d3ad6f3a336b7..d80b61cdf50e233 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -313,7 +313,8 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3706 +/* 3707: experimental PEP 671 query opcodes and function metadata. */ +#define PYC_MAGIC_NUMBER 3707 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d3546119bc9ee0e..854ae3243dfeac7 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -414,6 +414,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case PUSH_NULL: return 0; + case QUERY_DEREF: + return 0; + case QUERY_FAST: + return 0; case RAISE_VARARGS: return oparg; case RERAISE: @@ -909,6 +913,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2; case PUSH_NULL: return 1; + case QUERY_DEREF: + return 1; + case QUERY_FAST: + return 1; case RAISE_VARARGS: return 0; case RERAISE: @@ -1282,6 +1290,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [POP_TOP] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [QUERY_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, + [QUERY_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, @@ -1504,6 +1514,8 @@ _PyOpcode_macro_expansion[256] = { [POP_TOP] = { .nuops = 1, .uops = { { _POP_TOP, OPARG_SIMPLE, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { _PUSH_EXC_INFO, OPARG_SIMPLE, 0 } } }, [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, OPARG_SIMPLE, 0 } } }, + [QUERY_DEREF] = { .nuops = 1, .uops = { { _QUERY_DEREF, OPARG_SIMPLE, 0 } } }, + [QUERY_FAST] = { .nuops = 1, .uops = { { _QUERY_FAST, OPARG_SIMPLE, 0 } } }, [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 1 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, OPARG_SIMPLE, 0 } } }, [RETURN_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, @@ -1741,6 +1753,8 @@ const char *_PyOpcode_OpName[267] = { [POP_TOP] = "POP_TOP", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [PUSH_NULL] = "PUSH_NULL", + [QUERY_DEREF] = "QUERY_DEREF", + [QUERY_FAST] = "QUERY_FAST", [RAISE_VARARGS] = "RAISE_VARARGS", [RERAISE] = "RERAISE", [RESERVED] = "RESERVED", @@ -1828,8 +1842,6 @@ const uint8_t _PyOpcode_Caches[256] = { PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [117] = 117, - [118] = 118, [119] = 119, [120] = 120, [121] = 121, @@ -2035,6 +2047,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [POP_TOP] = POP_TOP, [PUSH_EXC_INFO] = PUSH_EXC_INFO, [PUSH_NULL] = PUSH_NULL, + [QUERY_DEREF] = QUERY_DEREF, + [QUERY_FAST] = QUERY_FAST, [RAISE_VARARGS] = RAISE_VARARGS, [RERAISE] = RERAISE, [RESERVED] = RESERVED, @@ -2089,8 +2103,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 117: \ - case 118: \ case 119: \ case 120: \ case 121: \ diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 7067b48ec22cb3c..1f3235a7e840c77 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -66,6 +66,8 @@ extern "C" { #define MAKE_FUNCTION_ANNOTATIONS 0x04 #define MAKE_FUNCTION_CLOSURE 0x08 #define MAKE_FUNCTION_ANNOTATE 0x10 +#define MAKE_FUNCTION_DEFAULTS_EXTRA 0x20 +#define MAKE_FUNCTION_KWDEFAULTS_EXTRA 0x40 /* Values used as the oparg for LOAD_COMMON_CONSTANT */ #define CONSTANT_ASSERTIONERROR 0 diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 34dc6228a0bcb08..252e943adb3b511 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1318,6 +1318,7 @@ extern "C" { INIT_STR(dbl_open_br, "{{"), \ INIT_STR(dbl_percent, "%%"), \ INIT_STR(defaults, ".defaults"), \ + INIT_STR(defaults_extra, ".defaults_extra"), \ INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ INIT_STR(format, ".format"), \ @@ -1325,6 +1326,7 @@ extern "C" { INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ + INIT_STR(kwdefaults_extra, ".kwdefaults_extra"), \ INIT_STR(list_err, "list index out of range"), \ INIT_STR(native, ""), \ INIT_STR(str_replace_inf, "1e309"), \ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 01afe5bb935b996..cb917751526a554 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -126,6 +126,7 @@ typedef struct _symtable_entry { unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */ unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an enclosing class scope */ + unsigned ste_has_late_defaults : 1; /* parameters initialized at call time */ unsigned ste_has_docstring : 1; /* true if docstring present */ unsigned ste_method : 1; /* true if block is a function block defined in class scope */ unsigned ste_has_conditional_annotations : 1; /* true if block has conditionally executed annotations */ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f2f3b9de4a0d87d..33cff0743398773 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -877,9 +877,11 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { offsetof(struct _Py_global_strings, literals._py_dbl_percent._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_dot_locals._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_defaults._ascii.ob_base), + offsetof(struct _Py_global_strings, literals._py_defaults_extra._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_format._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_generic_base._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_kwdefaults._ascii.ob_base), + offsetof(struct _Py_global_strings, literals._py_kwdefaults_extra._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_type_params._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_str_replace_inf._ascii.ob_base), offsetof(struct _Py_global_strings, literals._py_gc._ascii.ob_base), diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ba6e0252024a6df..621a985b4862992 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -427,6 +427,8 @@ enum { #define _POP_TOP POP_TOP #define _PUSH_EXC_INFO PUSH_EXC_INFO #define _PUSH_NULL PUSH_NULL +#define _QUERY_DEREF QUERY_DEREF +#define _QUERY_FAST QUERY_FAST #define _RETURN_GENERATOR RETURN_GENERATOR #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD @@ -1290,6 +1292,10 @@ enum { _PY_FRAME_EX_r31, _PY_FRAME_GENERAL_r01, _PY_FRAME_KW_r11, + _QUERY_DEREF_r01, + _QUERY_FAST_r01, + _QUERY_FAST_r12, + _QUERY_FAST_r23, _REPLACE_WITH_TRUE_r02, _REPLACE_WITH_TRUE_r12, _REPLACE_WITH_TRUE_r23, @@ -1444,7 +1450,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1655 +#define MAX_UOP_REGS_ID 1659 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 86b449158f00d0d..6a47a57ee3f33cb 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -39,6 +39,8 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, + [_QUERY_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, + [_QUERY_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, @@ -483,6 +485,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _RESUME_CHECK_r33 }, }, }, + [_QUERY_FAST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _QUERY_FAST_r01 }, + { 2, 1, _QUERY_FAST_r12 }, + { 3, 2, _QUERY_FAST_r23 }, + { -1, -1, -1 }, + }, + }, + [_QUERY_DEREF] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _QUERY_DEREF_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_LOAD_FAST_CHECK] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -3925,6 +3945,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_RESUME_CHECK_r11] = _RESUME_CHECK, [_RESUME_CHECK_r22] = _RESUME_CHECK, [_RESUME_CHECK_r33] = _RESUME_CHECK, + [_QUERY_FAST_r01] = _QUERY_FAST, + [_QUERY_FAST_r12] = _QUERY_FAST, + [_QUERY_FAST_r23] = _QUERY_FAST, + [_QUERY_DEREF_r01] = _QUERY_DEREF, [_LOAD_FAST_CHECK_r01] = _LOAD_FAST_CHECK, [_LOAD_FAST_CHECK_r12] = _LOAD_FAST_CHECK, [_LOAD_FAST_CHECK_r23] = _LOAD_FAST_CHECK, @@ -5974,6 +5998,12 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_PY_FRAME_GENERAL_r01] = "_PY_FRAME_GENERAL_r01", [_PY_FRAME_KW] = "_PY_FRAME_KW", [_PY_FRAME_KW_r11] = "_PY_FRAME_KW_r11", + [_QUERY_DEREF] = "_QUERY_DEREF", + [_QUERY_DEREF_r01] = "_QUERY_DEREF_r01", + [_QUERY_FAST] = "_QUERY_FAST", + [_QUERY_FAST_r01] = "_QUERY_FAST_r01", + [_QUERY_FAST_r12] = "_QUERY_FAST_r12", + [_QUERY_FAST_r23] = "_QUERY_FAST_r23", [_RECORD_3OS_GEN_FUNC] = "_RECORD_3OS_GEN_FUNC", [_RECORD_4OS] = "_RECORD_4OS", [_RECORD_BOUND_METHOD] = "_RECORD_BOUND_METHOD", @@ -6206,6 +6236,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _RESUME_CHECK: return 0; + case _QUERY_FAST: + return 0; + case _QUERY_DEREF: + return 0; case _LOAD_FAST_CHECK: return 0; case _LOAD_FAST_0: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 11342ae451b9f6c..7de458f536ec811 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -110,23 +110,25 @@ extern "C" { #define POP_JUMP_IF_NONE 97 #define POP_JUMP_IF_NOT_NONE 98 #define POP_JUMP_IF_TRUE 99 -#define RAISE_VARARGS 100 -#define RERAISE 101 -#define SEND 102 -#define SET_ADD 103 -#define SET_FUNCTION_ATTRIBUTE 104 -#define SET_UPDATE 105 -#define STORE_ATTR 106 -#define STORE_DEREF 107 -#define STORE_FAST 108 -#define STORE_FAST_LOAD_FAST 109 -#define STORE_FAST_STORE_FAST 110 -#define STORE_GLOBAL 111 -#define STORE_NAME 112 -#define SWAP 113 -#define UNPACK_EX 114 -#define UNPACK_SEQUENCE 115 -#define YIELD_VALUE 116 +#define QUERY_DEREF 100 +#define QUERY_FAST 101 +#define RAISE_VARARGS 102 +#define RERAISE 103 +#define SEND 104 +#define SET_ADD 105 +#define SET_FUNCTION_ATTRIBUTE 106 +#define SET_UPDATE 107 +#define STORE_ATTR 108 +#define STORE_DEREF 109 +#define STORE_FAST 110 +#define STORE_FAST_LOAD_FAST 111 +#define STORE_FAST_STORE_FAST 112 +#define STORE_GLOBAL 113 +#define STORE_NAME 114 +#define SWAP 115 +#define UNPACK_EX 116 +#define UNPACK_SEQUENCE 117 +#define YIELD_VALUE 118 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index 58bc06397f5134b..2b332d1417d73fc 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -1012,7 +1012,7 @@ def visit_arguments(self, node): self.write(", ") self.traverse(a) if d: - self.write("=") + self.write("=>" if a.late_default is not None else "=") self.traverse(d) if index == len(node.posonlyargs): self.write(", /") @@ -1036,7 +1036,7 @@ def visit_arguments(self, node): self.write(", ") self.traverse(a) if d: - self.write("=") + self.write("=>" if a.late_default is not None else "=") self.traverse(d) # kwargs diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index df92eae151d2488..74416cbb51331d7 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -330,23 +330,25 @@ POP_JUMP_IF_NONE=97, POP_JUMP_IF_NOT_NONE=98, POP_JUMP_IF_TRUE=99, - RAISE_VARARGS=100, - RERAISE=101, - SEND=102, - SET_ADD=103, - SET_FUNCTION_ATTRIBUTE=104, - SET_UPDATE=105, - STORE_ATTR=106, - STORE_DEREF=107, - STORE_FAST=108, - STORE_FAST_LOAD_FAST=109, - STORE_FAST_STORE_FAST=110, - STORE_GLOBAL=111, - STORE_NAME=112, - SWAP=113, - UNPACK_EX=114, - UNPACK_SEQUENCE=115, - YIELD_VALUE=116, + QUERY_DEREF=100, + QUERY_FAST=101, + RAISE_VARARGS=102, + RERAISE=103, + SEND=104, + SET_ADD=105, + SET_FUNCTION_ATTRIBUTE=106, + SET_UPDATE=107, + STORE_ATTR=108, + STORE_DEREF=109, + STORE_FAST=110, + STORE_FAST_LOAD_FAST=111, + STORE_FAST_STORE_FAST=112, + STORE_GLOBAL=113, + STORE_NAME=114, + SWAP=115, + UNPACK_EX=116, + UNPACK_SEQUENCE=117, + YIELD_VALUE=118, INSTRUMENTED_END_FOR=233, INSTRUMENTED_POP_ITER=234, INSTRUMENTED_END_SEND=235, diff --git a/Lib/dis.py b/Lib/dis.py index 318729091fa098c..87b29d16123aad0 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -32,7 +32,8 @@ CONVERT_VALUE = opmap['CONVERT_VALUE'] SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] -FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate') +FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure', 'annotate', + 'defaults_extra', 'kwdefaults_extra') ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] IMPORT_NAME = opmap['IMPORT_NAME'] diff --git a/Lib/inspect.py b/Lib/inspect.py index bceed35a53abba1..75183e36dc1e86e 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2363,6 +2363,8 @@ def _signature_from_function(cls, func, skip_bound_arg=True, format=annotation_format) defaults = func.__defaults__ kwdefaults = func.__kwdefaults__ + defaults_extra = getattr(func, "__defaults_extra__", None) or () + kwdefaults_extra = getattr(func, "__kwdefaults_extra__", None) or {} if defaults: pos_default_count = len(defaults) @@ -2387,9 +2389,10 @@ def _signature_from_function(cls, func, skip_bound_arg=True, for offset, name in enumerate(positional[non_default_count:]): kind = _POSITIONAL_ONLY if posonly_left else _POSITIONAL_OR_KEYWORD annotation = annotations.get(name, _empty) + extra = defaults_extra[offset] if offset < len(defaults_extra) else None + options = {'late_default': extra} if extra is not None else {} parameters.append(Parameter(name, annotation=annotation, - kind=kind, - default=defaults[offset])) + kind=kind, default=defaults[offset], **options)) if posonly_left: posonly_left -= 1 @@ -2407,9 +2410,10 @@ def _signature_from_function(cls, func, skip_bound_arg=True, default = kwdefaults.get(name, _empty) annotation = annotations.get(name, _empty) + extra = kwdefaults_extra.get(name) + options = {'late_default': extra} if extra is not None else {} parameters.append(Parameter(name, annotation=annotation, - kind=_KEYWORD_ONLY, - default=default)) + kind=_KEYWORD_ONLY, default=default, **options)) # **kwargs if func_code.co_flags & CO_VARKEYWORDS: index = pos_count + keyword_only_count @@ -2683,6 +2687,9 @@ class Parameter: The default value for the parameter if specified. If the parameter has no default value, this attribute is set to `Parameter.empty`. + * late_default : str | None + The source expression for a late-bound default, or None for an + early-bound default or a required parameter. * annotation The annotation for the parameter if specified. If the parameter has no annotation, this attribute is set to @@ -2695,7 +2702,7 @@ class Parameter: Every value has a `description` attribute describing meaning. """ - __slots__ = ('_name', '_kind', '_default', '_annotation') + __slots__ = ('_name', '_kind', '_default', '_annotation', '_late_default') POSITIONAL_ONLY = _POSITIONAL_ONLY POSITIONAL_OR_KEYWORD = _POSITIONAL_OR_KEYWORD @@ -2705,7 +2712,8 @@ class Parameter: empty = _empty - def __init__(self, name, kind, *, default=_empty, annotation=_empty): + def __init__(self, name, kind, *, default=_empty, annotation=_empty, + late_default=None): try: self._kind = _ParameterKind(kind) except ValueError: @@ -2715,6 +2723,7 @@ def __init__(self, name, kind, *, default=_empty, annotation=_empty): msg = '{} parameters cannot have default values' msg = msg.format(self._kind.description) raise ValueError(msg) + self._late_default = late_default self._default = default self._annotation = annotation @@ -2749,13 +2758,16 @@ def __init__(self, name, kind, *, default=_empty, annotation=_empty): self._name = name @classmethod - def _from_code(cls, name, kind, *, default=_empty, annotation=_empty): + def _from_code(cls, name, kind, *, default=_empty, annotation=_empty, + late_default=None): # Fast path for Python functions: only the name needs validation. if iskeyword(name) or not name.isidentifier(): - return cls(name, kind, default=default, annotation=annotation) + options = {'late_default': late_default} if late_default is not None else {} + return cls(name, kind, default=default, annotation=annotation, **options) self = object.__new__(cls) self._name = name self._kind = kind + self._late_default = late_default self._default = default self._annotation = annotation return self @@ -2763,10 +2775,11 @@ def _from_code(cls, name, kind, *, default=_empty, annotation=_empty): def __reduce__(self): return (type(self), (self._name, self._kind), - {'_default': self._default, + {'_default': self._default, '_late_default': self._late_default, '_annotation': self._annotation}) def __setstate__(self, state): + self._late_default = state.get('_late_default') self._default = state['_default'] self._annotation = state['_annotation'] @@ -2778,6 +2791,11 @@ def name(self): def default(self): return self._default + @property + def late_default(self): + """Source expression for a late-bound default, or None.""" + return self._late_default + @property def annotation(self): return self._annotation @@ -2787,7 +2805,7 @@ def kind(self): return self._kind def replace(self, *, name=_void, kind=_void, - annotation=_void, default=_void): + annotation=_void, default=_void, late_default=_void): """Creates a customized copy of the Parameter.""" if name is _void: @@ -2802,7 +2820,11 @@ def replace(self, *, name=_void, kind=_void, if default is _void: default = self._default - return type(self)(name, kind, default=default, annotation=annotation) + if late_default is _void: + late_default = self._late_default + + options = {'late_default': late_default} if late_default is not None else {} + return type(self)(name, kind, default=default, annotation=annotation, **options) def __str__(self): return self._format() @@ -2817,7 +2839,9 @@ def _format(self, *, quote_annotation_strings=True): quote_annotation_strings=quote_annotation_strings) formatted = '{}: {}'.format(formatted, annotation) - if self._default is not _empty: + if self._late_default is not None and self._default is Ellipsis: + formatted += f"=>{self._late_default}" + elif self._default is not _empty: if self._annotation is not _empty: formatted = '{} = {}'.format(formatted, repr(self._default)) else: @@ -2836,7 +2860,7 @@ def __repr__(self): return '<{} "{}">'.format(self.__class__.__name__, self) def __hash__(self): - return hash((self._name, self._kind, self._annotation, self._default)) + return hash((self._name, self._kind, self._annotation, self._default, self._late_default)) def __eq__(self, other): if self is other: @@ -2846,7 +2870,8 @@ def __eq__(self, other): return (self._name == other._name and self._kind == other._kind and self._default == other._default and - self._annotation == other._annotation) + self._annotation == other._annotation and + self._late_default == other._late_default) class BoundArguments: diff --git a/Lib/test/test_ast/snippets.py b/Lib/test/test_ast/snippets.py index a565ed10f8b434b..20970b0527ceb92 100644 --- a/Lib/test/test_ast/snippets.py +++ b/Lib/test/test_ast/snippets.py @@ -398,14 +398,14 @@ def main(): ('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), ('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), ('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None, None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None, None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None, None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None, None), ('arg', (1, 9, 1, 10), 'b', None, None, None), ('arg', (1, 14, 1, 15), 'c', None, None, None), ('arg', (1, 22, 1, 23), 'd', None, None, None), ('arg', (1, 28, 1, 29), 'e', None, None, None)], ('arg', (1, 35, 1, 39), 'args', None, None, None), [('arg', (1, 41, 1, 42), 'f', None, None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), ('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), ('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), ('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), @@ -503,16 +503,16 @@ def main(): ('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), ('Module', [('If', (1, 0, 1, 19), ('NamedExpr', (1, 3, 1, 13), ('Name', (1, 3, 1, 4), 'a', ('Store',)), ('Call', (1, 8, 1, 13), ('Name', (1, 8, 1, 11), 'foo', ('Load',)), [], [])), [('Pass', (1, 15, 1, 19))], [])], []), ('Module', [('While', (1, 0, 1, 22), ('NamedExpr', (1, 6, 1, 16), ('Name', (1, 6, 1, 7), 'a', ('Store',)), ('Call', (1, 11, 1, 16), ('Name', (1, 11, 1, 14), 'foo', ('Load',)), [], [])), [('Pass', (1, 18, 1, 22))], [])], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None, None)], [('arg', (1, 12, 1, 13), 'c', None, None, None), ('arg', (1, 15, 1, 16), 'd', None, None, None), ('arg', (1, 18, 1, 19), 'e', None, None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None, None)], [('arg', (1, 12, 1, 13), 'c', None, None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None, None), ('arg', (1, 21, 1, 22), 'e', None, None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None, None)], [('arg', (1, 12, 1, 13), 'c', None, None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None, None), ('arg', (1, 21, 1, 22), 'e', None, None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None, None)], [('arg', (1, 14, 1, 15), 'b', None, None, None), ('arg', (1, 19, 1, 20), 'c', None, None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None, None)], [('arg', (1, 14, 1, 15), 'b', None, None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None, None)], [('arg', (1, 14, 1, 15), 'b', None, None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None, None)], [('arg', (1, 14, 1, 15), 'b', None, None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None, None)], [('arg', (1, 14, 1, 15), 'b', None, None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), ('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), ('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), ('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts', None), ('ParamSpec', (1, 15, 1, 18), 'P', None)], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), diff --git a/Lib/test/test_late_defaults.py b/Lib/test/test_late_defaults.py new file mode 100644 index 000000000000000..b8571c39e9d774c --- /dev/null +++ b/Lib/test/test_late_defaults.py @@ -0,0 +1,406 @@ +"""Behavioral tests for the experimental PEP 671 implementation.""" + +import ast +import copy +import functools +import gc +import inspect +import marshal +import pickle +import unittest +import weakref + + +class LateDefaultsTests(unittest.TestCase): + def test_fresh_mutable_default(self): + def add(item, target=>[]): + target.append(item) + return target + a, b = add(1), add(2) + self.assertEqual(a, [1]) + self.assertEqual(b, [2]) + self.assertIsNot(a, b) + target = [] + self.assertIs(add(3, target), target) + + def test_definition_does_not_evaluate(self): + calls = [] + def f(x=>calls.append('default')): + return x + self.assertEqual(calls, []) + self.assertEqual(f(12), 12) + self.assertEqual(calls, []) + self.assertIsNone(f()) + self.assertEqual(calls, ['default']) + + def test_explicit_values_including_ellipsis(self): + def f(a=>1, *, b=>2): + return a, b + for value in [None, False, 0, '', [], Ellipsis]: + with self.subTest(value=value): + a, b = f(value, b=value) + self.assertIs(a, value) + self.assertIs(b, value) + self.assertEqual(f(), (1, 2)) + + def test_left_to_right(self): + calls = [] + def mark(n): + calls.append(n) + return n + def f(a=>mark(1), b=>mark(a + 1), *, c=>mark(b + 1)): + return a, b, c + self.assertEqual(f(), (1, 2, 3)) + self.assertEqual(calls, [1, 2, 3]) + calls.clear() + self.assertEqual(f(c=20, a=4), (4, 5, 20)) + self.assertEqual(calls, [5]) + + def test_mixed_binding_times(self): + calls = [] + def f(a=calls.append('early'), b=>calls.append('late')): + return a, b + self.assertEqual(calls, ['early']) + self.assertEqual(f(), (None, None)) + self.assertEqual(f(), (None, None)) + self.assertEqual(calls, ['early', 'late', 'late']) + + def test_parameter_kinds(self): + def f(a, b=>len(a), /, c=>b + 1, *rest, d=>c + 1, **kw): + return b, c, rest, d, kw + self.assertEqual(f([1, 2]), (2, 3, (), 4, {})) + self.assertEqual(f([], 5, 8, 9, d=10, x=11), (5, 8, (9,), 10, {'x': 11})) + self.assertEqual(f([], **{'b': 30}), (0, 1, (), 2, {'b': 30})) + + def test_keyword_only_without_default(self): + def f(*, required, x=>required + 1): + return x + self.assertEqual(f(required=3), 4) + with self.assertRaises(TypeError): + f() + + def test_runtime_scope(self): + x = 100 + def f(x, y=>x): + return y + self.assertEqual(f(2), 2) + def g(x=>x): + return x + with self.assertRaises(UnboundLocalError): + g() + self.assertEqual(g(3), 3) + + def test_forward_reference(self): + def f(a=>b, b=>2): + return a, b + with self.assertRaises(UnboundLocalError): + f() + self.assertEqual(f(b=9), (9, 9)) + def g(a=>b, b=4): + return a, b + self.assertEqual(g(), (4, 4)) + + def test_closure_is_read_at_call_time(self): + x = 1 + def f(a=>x): + return a + self.assertEqual(f(), 1) + x = 2 + self.assertEqual(f(), 2) + + def test_cell_parameter(self): + def f(a=>[], b=>a): + def inner(): + return a + return inner, b + inner, b = f() + self.assertIs(inner(), b) + value = object() + inner, b = f(value) + self.assertIs(inner(), value) + self.assertIs(b, value) + + def test_self_reference_cell(self): + def f(a=>a): + return lambda: a + with self.assertRaises(UnboundLocalError): + f() + self.assertEqual(f(4)(), 4) + + def test_lambdas_and_nested_scopes(self): + f = lambda a=>[], *, b=>len(a): (a, b) + self.assertEqual(f(), ([], 0)) + self.assertEqual(f([1, 2]), ([1, 2], 2)) + def g(a=>3, b=>lambda: a): + return b() + self.assertEqual(g(), 3) + + def test_comprehension(self): + def f(n, values=>[i * n for i in range(n)]): + return values + self.assertEqual(f(3), [0, 3, 6]) + + def test_exception(self): + def f(x=>1 / 0): + return x + with self.assertRaises(ZeroDivisionError): + f() + self.assertEqual(f(3), 3) + + def test_argument_errors_precede_evaluation(self): + calls = [] + def f(required, x=>calls.append(1)): + return x + for args, kw in [((), {}), ((1, 2, 3), {}), ((1,), {'unknown': 2}), + ((1, 2), {'x': 3})]: + with self.subTest(args=args, kw=kw): + with self.assertRaises(TypeError): + f(*args, **kw) + self.assertEqual(calls, []) + + def test_method_and_mangling(self): + class C: + def f(self, __x=>4, *, __y=>5): + return __x, __y + def owner(self, cls=>__class__): + return cls + c = C() + self.assertEqual(c.f(), (4, 5)) + self.assertEqual(c.f(_C__x=6, _C__y=7), (6, 7)) + self.assertIs(c.owner(), C) + + def test_generic(self): + def f[T](x: T, items=>[x]): + return items + self.assertEqual(f(3), [3]) + + def test_docstring(self): + def f(x=>'default'): + """Function documentation.""" + return x + self.assertEqual(f.__doc__, 'Function documentation.') + self.assertEqual(f(), 'default') + + def test_coroutine(self): + async def f(x=>[]): + return x + def run(): + coro = f() + try: + coro.send(None) + except StopIteration as e: + return e.value + finally: + coro.close() + a, b = run(), run() + self.assertEqual(a, []) + self.assertIsNot(a, b) + + def test_generator(self): + def f(x=>[]): + yield x + a, b = next(f()), next(f()) + self.assertEqual(a, []) + self.assertIsNot(a, b) + + def test_specialized_call_sites(self): + def f(x=>[], *, y=>[]): + return x, y + def call(): + return f() + for _ in range(1000): + a, b = call() + self.assertEqual(a, []) + self.assertIsNot(a, b) + + def test_c_vectorcall(self): + from test.support import import_helper + _testcapi = import_helper.import_module('_testcapi') + def f(a=>[], *, b=>{}): + return a, b + self.assertEqual(_testcapi.pyobject_vectorcall(f, None, None), ([], {})) + value = object() + a, b = _testcapi.pyobject_vectorcall(f, (value, value), ('b',)) + self.assertIs(a, value) + self.assertIs(b, value) + + def test_missing_keyword_error_names(self): + def f(*, a, b=>[], c): + pass + with self.assertRaisesRegex(TypeError, "2 required keyword-only arguments: 'a' and 'c'"): + f() + + def test_generic_with_both_default_kinds(self): + def f[T](a=1, b=>a + 1, *, c=3, d=>b + c): + return a, b, c, d + self.assertEqual(f(), (1, 2, 3, 5)) + self.assertEqual(f(5, d=20), (5, 6, 3, 20)) + + def test_keyword_cell_parameter(self): + def f(*, x=>[]): + return lambda: x + a, b = f()(), f()() + self.assertEqual(a, []) + self.assertIsNot(a, b) + + def test_partial(self): + def f(a=>1, b=>a + 1): + return a, b + self.assertEqual(functools.partial(f, b=9)(), (1, 9)) + self.assertEqual(functools.partial(f, 4)(), (4, 5)) + + def test_many_parameters(self): + names = [f'x{i}' for i in range(75)] + namespace = {} + exec('def f(' + ', '.join(n + '=>[]' for n in names) + '):\n' + ' return (' + ', '.join(names) + ')', namespace) + result = namespace['f']() + self.assertEqual(len(result), 75) + self.assertTrue(all(value == [] for value in result)) + self.assertEqual(len({id(value) for value in result}), 75) + + def test_metadata_and_signature(self): + def f(a=..., b=>[], *, c=>len(b), d=...): + pass + self.assertEqual(f.__defaults__, (..., ...)) + self.assertEqual(f.__defaults_extra__, (None, '[]')) + self.assertEqual(f.__kwdefaults_extra__, {'c': 'len(b)'}) + sig = inspect.signature(f) + self.assertEqual(str(sig), '(a=Ellipsis, b=>[], *, c=>len(b), d=Ellipsis)') + self.assertEqual(sig.parameters['b'].late_default, '[]') + self.assertIsNone(sig.parameters['a'].late_default) + self.assertEqual(len(inspect.getfullargspec(f)), 7) + + def test_signature_copy_and_pickle(self): + def f(x=>[]): + pass + sig = inspect.signature(f) + parameter = sig.parameters['x'] + self.assertEqual(parameter.replace(), parameter) + self.assertEqual(copy.copy(parameter), parameter) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertEqual(pickle.loads(pickle.dumps(sig, protocol)), sig) + early = parameter.replace(late_default=None) + self.assertNotEqual(early, parameter) + + def test_mutated_defaults(self): + def f(x=>[], *, y=>{}): + return x, y + f.__defaults__ = (7,) + f.__kwdefaults__ = {'y': 8} + self.assertEqual(f(), (7, 8)) + f.__defaults__ = (...,) + f.__kwdefaults__ = {'y': ...} + self.assertEqual(f(), ([], {})) + f.__defaults_extra__ = None + f.__kwdefaults_extra__ = None + self.assertEqual(f(), (..., ...)) + + def test_metadata_type_checks(self): + def f(x=>[]): + pass + with self.assertRaises(TypeError): + f.__defaults_extra__ = [] + with self.assertRaises(TypeError): + f.__kwdefaults_extra__ = () + del f.__defaults_extra__ + self.assertIsNone(f.__defaults_extra__) + + def test_ast_and_unparse(self): + source = 'def f(a=>[], /, b=1, *, c=>len(a)):\n return a, b, c' + tree = ast.parse(source) + args = tree.body[0].args + self.assertEqual(args.posonlyargs[0].late_default, '[]') + self.assertIsNone(args.args[0].late_default) + self.assertEqual(args.kwonlyargs[0].late_default, 'len(a)') + namespace = {} + exec(compile(tree, '', 'exec'), namespace) + self.assertEqual(namespace['f'](), ([], 1, 0)) + unparsed = ast.unparse(tree) + self.assertIn('a=>[]', unparsed) + self.assertIn('c=>len(a)', unparsed) + self.assertTrue(ast.compare(tree, ast.parse(unparsed))) + expression = ast.parse('lambda x=>[]: x', mode='eval') + self.assertEqual(ast.unparse(expression), 'lambda x=>[]: x') + + def test_invalid_ast(self): + tree = ast.parse('def f(x): pass') + tree.body[0].args.args[0].late_default = '[]' + with self.assertRaisesRegex(ValueError, 'no default'): + compile(tree, '', 'exec') + tree = ast.parse('def f(*args): pass') + tree.body[0].args.vararg.late_default = '[]' + with self.assertRaisesRegex(ValueError, 'variadic'): + compile(tree, '', 'exec') + + def test_source_survives_constant_folding(self): + def f(x=>1 + 2, *, y=>3 * 4): + return x, y + self.assertEqual(f(), (3, 12)) + self.assertEqual(f.__defaults_extra__, ('1 + 2',)) + self.assertEqual(f.__kwdefaults_extra__, {'y': '3 * 4'}) + + def test_metadata_on_ordinary_function(self): + def f(x=..., *, y=...): + return x, y + f.__defaults_extra__ = ('[]',) + f.__kwdefaults_extra__ = {'y': '{}'} + self.assertEqual(f(), (..., ...)) + + def test_code_replacement(self): + def f(x=>[]): + return x + def g(x): + return x + f.__code__ = g.__code__ + self.assertIs(f(), Ellipsis) + + def test_type_comment(self): + tree = ast.parse('def f(x=>[], # type: list\n): pass', type_comments=True) + self.assertEqual(tree.body[0].args.args[0].late_default, '[]') + self.assertEqual(tree.body[0].args.args[0].type_comment, 'list') + + def test_invalid_syntax(self): + for source in ['def f(x=>): pass', 'def f(x=>1, y): pass', + 'def f(*args=>[]): pass', 'def f(**kw=>{}): pass', + 'lambda x=>: x', 'lambda x=>1, y: x', + 'lambda x: int=>1, y: x']: + with self.subTest(source=source), self.assertRaises(SyntaxError): + compile(source, '', 'exec') + + def test_marshal_roundtrip(self): + namespace = {} + code = compile('def f(x=>[]): return x', '', 'exec') + exec(marshal.loads(marshal.dumps(code)), namespace) + a, b = namespace['f'](), namespace['f']() + self.assertEqual(a, []) + self.assertIsNot(a, b) + + def test_concurrent_calls(self): + from test.support import threading_helper + threading_helper.requires_working_threading()(lambda: None)() + import threading + results = [] + def f(x=>[], *, y=>{}): + return x, y + def worker(): + results.append([f() for _ in range(100)]) + threads = [threading.Thread(target=worker) for _ in range(4)] + with threading_helper.start_threads(threads): + pass + values = [value for group in results for pair in group for value in pair] + self.assertEqual(len(values), 800) + self.assertEqual(len({id(value) for value in values}), 800) + + def test_gc_cycle(self): + def create(): + def f(x=>f): + return x + return weakref.ref(f) + ref = create() + gc.collect() + self.assertIsNone(ref()) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index af99437821d2c36..f9efbc52f1fe53e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1815,7 +1815,7 @@ def func(): check(x, size('3PiccPPPP' + INTERPRETER_FRAME + 'P')) # function def func(): pass - check(func, size('16Pi')) + check(func, size('18Pi')) class c(): @staticmethod def foo(): diff --git a/Misc/NEWS.d/next/Core and Builtins/2026-10-15-17-00-00.pep-671.rst b/Misc/NEWS.d/next/Core and Builtins/2026-10-15-17-00-00.pep-671.rst new file mode 100644 index 000000000000000..57d41ae4126324d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2026-10-15-17-00-00.pep-671.rst @@ -0,0 +1,3 @@ +Add an experimental implementation of :pep:`671` late-bound argument defaults +using ``name=>expression``. This is a reference implementation of a draft PEP; +the proposal has not been accepted. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 3bdc16437e2bc61..878fa944e64a7f0 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -11144,6 +11144,45 @@ DISPATCH(); } + TARGET(QUERY_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = QUERY_DEREF; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(QUERY_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *contents = PyCell_GetRef(cell); + value = contents == NULL ? PyStackRef_False : PyStackRef_True; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_XDECREF(contents); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(QUERY_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = QUERY_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(QUERY_FAST); + _PyStackRef value; + value = PyStackRef_IsNull(GETLOCAL(oparg)) + ? PyStackRef_False : PyStackRef_True; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + TARGET(RAISE_VARARGS) { #if _Py_TAIL_CALL_INTERP int opcode = RAISE_VARARGS; diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4b..4d3d67d8457d687 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -100,6 +100,8 @@ static void *opcode_targets_table[256] = { &&TARGET_POP_JUMP_IF_NONE, &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_TRUE, + &&TARGET_QUERY_DEREF, + &&TARGET_QUERY_FAST, &&TARGET_RAISE_VARARGS, &&TARGET_RERAISE, &&TARGET_SEND, @@ -126,8 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -376,8 +376,8 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -713,6 +713,8 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAM static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_QUERY_DEREF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_QUERY_FAST(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); @@ -956,6 +958,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [POP_TOP] = _TAIL_CALL_POP_TOP, [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, + [QUERY_DEREF] = _TAIL_CALL_QUERY_DEREF, + [QUERY_FAST] = _TAIL_CALL_QUERY_FAST, [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, [RERAISE] = _TAIL_CALL_RERAISE, [RESERVED] = _TAIL_CALL_RESERVED, @@ -1005,8 +1009,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1214,6 +1216,8 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [POP_TOP] = _TAIL_CALL_TRACE_RECORD, [PUSH_EXC_INFO] = _TAIL_CALL_TRACE_RECORD, [PUSH_NULL] = _TAIL_CALL_TRACE_RECORD, + [QUERY_DEREF] = _TAIL_CALL_TRACE_RECORD, + [QUERY_FAST] = _TAIL_CALL_TRACE_RECORD, [RAISE_VARARGS] = _TAIL_CALL_TRACE_RECORD, [RERAISE] = _TAIL_CALL_TRACE_RECORD, [RESERVED] = _TAIL_CALL_TRACE_RECORD, @@ -1263,8 +1267,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 4d3da234f87e494..0efdd1a2a6b52a9 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -134,6 +134,8 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) _Py_INCREF_CODE((PyCodeObject *)constr->fc_code); op->func_code = constr->fc_code; op->func_defaults = Py_XNewRef(constr->fc_defaults); + op->func_defaults_extra = NULL; + op->func_kwdefaults_extra = NULL; op->func_kwdefaults = Py_XNewRef(constr->fc_kwdefaults); op->func_closure = Py_XNewRef(constr->fc_closure); op->func_doc = Py_NewRef(Py_None); @@ -212,6 +214,8 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_qualname = qualname; op->func_code = (PyObject*)code_obj; op->func_defaults = NULL; // No default positional arguments + op->func_defaults_extra = NULL; + op->func_kwdefaults_extra = NULL; op->func_kwdefaults = NULL; // No default keyword arguments op->func_closure = NULL; op->func_doc = doc; @@ -369,6 +373,10 @@ _PyFunction_ClearCodeByVersion(uint32_t version) uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) { + /* Existing call specializations assume that defaults are already bound. */ + if (((PyCodeObject *)func->func_code)->co_flags & CO_LATE_DEFAULTS) { + return FUNC_VERSION_UNSET; + } return func->func_version; } @@ -905,6 +913,99 @@ func_set_kwdefaults(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) return 0; } +static PyObject * +func_get_defaults_extra(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFunctionObject *op = _PyFunction_CAST(self); + if (PySys_Audit("object.__getattr__", "Os", op, "__defaults_extra__") < 0) { + return NULL; + } + if (op->func_defaults_extra == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(op->func_defaults_extra); +} + +static int +func_set_defaults_extra(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) +{ + /* Legal to del f.func_defaults_extra. + * Can only set func_defaults_extra to NULL or a tuple. */ + PyFunctionObject *op = _PyFunction_CAST(self); + if (value == Py_None) + value = NULL; + if (value != NULL && !PyTuple_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__defaults_extra__ must be set to a tuple object"); + return -1; + } + if (value) { + if (PySys_Audit("object.__setattr__", "OsO", + op, "__defaults_extra__", value) < 0) { + return -1; + } + } else if (PySys_Audit("object.__delattr__", "Os", + op, "__defaults_extra__") < 0) { + return -1; + } + + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + func_clear_version(interp, op); + PyObject *old_defaults = op->func_defaults_extra; + op->func_defaults_extra = Py_XNewRef(value); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_defaults); + return 0; +} + +static PyObject * +func_get_kwdefaults_extra(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyFunctionObject *op = _PyFunction_CAST(self); + if (PySys_Audit("object.__getattr__", "Os", + op, "__kwdefaults_extra__") < 0) { + return NULL; + } + if (op->func_kwdefaults_extra == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(op->func_kwdefaults_extra); +} + +static int +func_set_kwdefaults_extra(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) +{ + PyFunctionObject *op = _PyFunction_CAST(self); + if (value == Py_None) + value = NULL; + /* Legal to del f.func_kwdefaults_extra. + * Can only set func_kwdefaults_extra to NULL or a dict. */ + if (value != NULL && !PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults_extra__ must be set to a dict object"); + return -1; + } + if (value) { + if (PySys_Audit("object.__setattr__", "OsO", + op, "__kwdefaults_extra__", value) < 0) { + return -1; + } + } else if (PySys_Audit("object.__delattr__", "Os", + op, "__kwdefaults_extra__") < 0) { + return -1; + } + + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + func_clear_version(interp, op); + PyObject *old_kwdefaults = op->func_kwdefaults_extra; + op->func_kwdefaults_extra = Py_XNewRef(value); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_kwdefaults); + return 0; +} + /*[clinic input] @critical_section @getter @@ -1048,6 +1149,8 @@ static PyGetSetDef func_getsetlist[] = { {"__code__", func_get_code, func_set_code}, {"__defaults__", func_get_defaults, func_set_defaults}, {"__kwdefaults__", func_get_kwdefaults, func_set_kwdefaults}, + {"__defaults_extra__", func_get_defaults_extra, func_set_defaults_extra}, + {"__kwdefaults_extra__", func_get_kwdefaults_extra, func_set_kwdefaults_extra}, FUNCTION___ANNOTATIONS___GETSETDEF FUNCTION___ANNOTATE___GETSETDEF {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, @@ -1184,6 +1287,8 @@ func_clear(PyObject *self) } Py_CLEAR(op->func_module); Py_CLEAR(op->func_defaults); + Py_CLEAR(op->func_defaults_extra); + Py_CLEAR(op->func_kwdefaults_extra); Py_CLEAR(op->func_kwdefaults); Py_CLEAR(op->func_doc); Py_CLEAR(op->func_dict); @@ -1237,6 +1342,8 @@ func_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(f->func_builtins); Py_VISIT(f->func_module); Py_VISIT(f->func_defaults); + Py_VISIT(f->func_defaults_extra); + Py_VISIT(f->func_kwdefaults_extra); Py_VISIT(f->func_kwdefaults); Py_VISIT(f->func_doc); Py_VISIT(f->func_name); diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 2f0b123858f8d18..1d331fa509b6ff4 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -116,7 +116,7 @@ module Python arguments = (arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr?* kw_defaults, arg? kwarg, expr* defaults) - arg = (identifier arg, expr? annotation, string? type_comment) + arg = (identifier arg, expr? annotation, string? type_comment, string? late_default) attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) -- keyword arguments supplied to call (NULL identifier for **kwargs) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 2d51795d887993d..a25035e8a7c7068 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -467,6 +467,18 @@ _PyPegen_name_default_pair(Parser *p, arg_ty arg, expr_ty value, Token *tc) return a; } +/* Preserve the expression for introspection before AST optimization. */ +NameDefaultPair * +_PyPegen_late_default_pair(Parser *p, arg_ty arg, expr_ty value, Token *tc) +{ + PyObject *source = _PyAST_ExprAsUnicode(value); + if (source == NULL || _PyArena_AddPyObject(p->arena, source) < 0) { + return NULL; + } + arg->late_default = source; + return _PyPegen_name_default_pair(p, arg, value, tc); +} + /* Constructs a SlashWithDefault */ SlashWithDefault * _PyPegen_slash_with_default(Parser *p, asdl_arg_seq *plain_names, asdl_seq *names_with_defaults) @@ -938,7 +950,7 @@ _PyPegen_add_type_comment_to_arg(Parser *p, arg_ty a, Token *tc) if (tco == NULL) { return NULL; } - return _PyAST_arg(a->arg, a->annotation, tco, + return _PyAST_arg(a->arg, a->annotation, tco, a->late_default, a->lineno, a->col_offset, a->end_lineno, a->end_col_offset, p->arena); } diff --git a/Parser/parser.c b/Parser/parser.c index 979907f4873b9dd..72e68d26b1e7766 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -5578,7 +5578,11 @@ param_no_default_star_annotation_rule(Parser *p) return _res; } -// param_with_default: param default ',' TYPE_COMMENT? | param default TYPE_COMMENT? &')' +// param_with_default: +// | param '=' '>' expression ',' TYPE_COMMENT? +// | param '=' '>' expression TYPE_COMMENT? &')' +// | param default ',' TYPE_COMMENT? +// | param default TYPE_COMMENT? &')' static NameDefaultPair* param_with_default_rule(Parser *p) { @@ -5591,6 +5595,83 @@ param_with_default_rule(Parser *p) } NameDefaultPair* _res = NULL; int _mark = p->mark; + { // param '=' '>' expression ',' TYPE_COMMENT? + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> param_with_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param '=' '>' expression ',' TYPE_COMMENT?")); + Token * _literal; + Token * _literal_1; + Token * _literal_2; + arg_ty a; + expr_ty c; + void *tc; + if ( + (a = param_rule(p)) // param + && + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (_literal_1 = _PyPegen_expect_token(p, 21)) // token='>' + && + (c = expression_rule(p)) // expression + && + (_literal_2 = _PyPegen_expect_token(p, 12)) // token=',' + && + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? + ) + { + D(fprintf(stderr, "%*c+ param_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param '=' '>' expression ',' TYPE_COMMENT?")); + _res = _PyPegen_late_default_pair ( p , a , c , tc ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s param_with_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param '=' '>' expression ',' TYPE_COMMENT?")); + } + { // param '=' '>' expression TYPE_COMMENT? &')' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> param_with_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param '=' '>' expression TYPE_COMMENT? &')'")); + Token * _literal; + Token * _literal_1; + arg_ty a; + expr_ty c; + void *tc; + if ( + (a = param_rule(p)) // param + && + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (_literal_1 = _PyPegen_expect_token(p, 21)) // token='>' + && + (c = expression_rule(p)) // expression + && + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? + && + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) // token=')' + ) + { + D(fprintf(stderr, "%*c+ param_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param '=' '>' expression TYPE_COMMENT? &')'")); + _res = _PyPegen_late_default_pair ( p , a , c , tc ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s param_with_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param '=' '>' expression TYPE_COMMENT? &')'")); + } { // param default ',' TYPE_COMMENT? if (p->error_indicator) { p->level--; @@ -5663,6 +5744,8 @@ param_with_default_rule(Parser *p) } // param_maybe_default: +// | param '=' '>' expression ',' TYPE_COMMENT? +// | param '=' '>' expression TYPE_COMMENT? &')' // | param default? ',' TYPE_COMMENT? // | param default? TYPE_COMMENT? &')' static NameDefaultPair* @@ -5677,6 +5760,83 @@ param_maybe_default_rule(Parser *p) } NameDefaultPair* _res = NULL; int _mark = p->mark; + { // param '=' '>' expression ',' TYPE_COMMENT? + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> param_maybe_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param '=' '>' expression ',' TYPE_COMMENT?")); + Token * _literal; + Token * _literal_1; + Token * _literal_2; + arg_ty a; + expr_ty c; + void *tc; + if ( + (a = param_rule(p)) // param + && + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (_literal_1 = _PyPegen_expect_token(p, 21)) // token='>' + && + (c = expression_rule(p)) // expression + && + (_literal_2 = _PyPegen_expect_token(p, 12)) // token=',' + && + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? + ) + { + D(fprintf(stderr, "%*c+ param_maybe_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param '=' '>' expression ',' TYPE_COMMENT?")); + _res = _PyPegen_late_default_pair ( p , a , c , tc ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s param_maybe_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param '=' '>' expression ',' TYPE_COMMENT?")); + } + { // param '=' '>' expression TYPE_COMMENT? &')' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> param_maybe_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param '=' '>' expression TYPE_COMMENT? &')'")); + Token * _literal; + Token * _literal_1; + arg_ty a; + expr_ty c; + void *tc; + if ( + (a = param_rule(p)) // param + && + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (_literal_1 = _PyPegen_expect_token(p, 21)) // token='>' + && + (c = expression_rule(p)) // expression + && + (tc = _PyPegen_expect_token(p, TYPE_COMMENT), !p->error_indicator) // TYPE_COMMENT? + && + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 8) // token=')' + ) + { + D(fprintf(stderr, "%*c+ param_maybe_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param '=' '>' expression TYPE_COMMENT? &')'")); + _res = _PyPegen_late_default_pair ( p , a , c , tc ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s param_maybe_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param '=' '>' expression TYPE_COMMENT? &')'")); + } { // param default? ',' TYPE_COMMENT? if (p->error_indicator) { p->level--; @@ -5794,7 +5954,7 @@ param_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_arg ( a -> v . Name . id , b , NULL , EXTRA ); + _res = _PyAST_arg ( a -> v . Name . id , b , NULL , NULL , EXTRA ); if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -5858,7 +6018,7 @@ param_star_annotation_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_arg ( a -> v . Name . id , b , NULL , EXTRA ); + _res = _PyAST_arg ( a -> v . Name . id , b , NULL , NULL , EXTRA ); if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -15963,7 +16123,11 @@ lambda_param_no_default_rule(Parser *p) return _res; } -// lambda_param_with_default: lambda_param default ',' | lambda_param default &':' +// lambda_param_with_default: +// | lambda_param '=' '>' expression ',' +// | lambda_param '=' '>' expression &':' +// | lambda_param default ',' +// | lambda_param default &':' static NameDefaultPair* lambda_param_with_default_rule(Parser *p) { @@ -15976,6 +16140,77 @@ lambda_param_with_default_rule(Parser *p) } NameDefaultPair* _res = NULL; int _mark = p->mark; + { // lambda_param '=' '>' expression ',' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> lambda_param_with_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param '=' '>' expression ','")); + Token * _literal; + Token * _literal_1; + Token * _literal_2; + arg_ty a; + expr_ty c; + if ( + (a = lambda_param_rule(p)) // lambda_param + && + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (_literal_1 = _PyPegen_expect_token(p, 21)) // token='>' + && + (c = expression_rule(p)) // expression + && + (_literal_2 = _PyPegen_expect_token(p, 12)) // token=',' + ) + { + D(fprintf(stderr, "%*c+ lambda_param_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param '=' '>' expression ','")); + _res = _PyPegen_late_default_pair ( p , a , c , NULL ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s lambda_param_with_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param '=' '>' expression ','")); + } + { // lambda_param '=' '>' expression &':' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> lambda_param_with_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param '=' '>' expression &':'")); + Token * _literal; + Token * _literal_1; + arg_ty a; + expr_ty c; + if ( + (a = lambda_param_rule(p)) // lambda_param + && + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (_literal_1 = _PyPegen_expect_token(p, 21)) // token='>' + && + (c = expression_rule(p)) // expression + && + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 11) // token=':' + ) + { + D(fprintf(stderr, "%*c+ lambda_param_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param '=' '>' expression &':'")); + _res = _PyPegen_late_default_pair ( p , a , c , NULL ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s lambda_param_with_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param '=' '>' expression &':'")); + } { // lambda_param default ',' if (p->error_indicator) { p->level--; @@ -16041,7 +16276,11 @@ lambda_param_with_default_rule(Parser *p) return _res; } -// lambda_param_maybe_default: lambda_param default? ',' | lambda_param default? &':' +// lambda_param_maybe_default: +// | lambda_param '=' '>' expression ',' +// | lambda_param '=' '>' expression &':' +// | lambda_param default? ',' +// | lambda_param default? &':' static NameDefaultPair* lambda_param_maybe_default_rule(Parser *p) { @@ -16054,6 +16293,77 @@ lambda_param_maybe_default_rule(Parser *p) } NameDefaultPair* _res = NULL; int _mark = p->mark; + { // lambda_param '=' '>' expression ',' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> lambda_param_maybe_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param '=' '>' expression ','")); + Token * _literal; + Token * _literal_1; + Token * _literal_2; + arg_ty a; + expr_ty c; + if ( + (a = lambda_param_rule(p)) // lambda_param + && + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (_literal_1 = _PyPegen_expect_token(p, 21)) // token='>' + && + (c = expression_rule(p)) // expression + && + (_literal_2 = _PyPegen_expect_token(p, 12)) // token=',' + ) + { + D(fprintf(stderr, "%*c+ lambda_param_maybe_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param '=' '>' expression ','")); + _res = _PyPegen_late_default_pair ( p , a , c , NULL ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s lambda_param_maybe_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param '=' '>' expression ','")); + } + { // lambda_param '=' '>' expression &':' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> lambda_param_maybe_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param '=' '>' expression &':'")); + Token * _literal; + Token * _literal_1; + arg_ty a; + expr_ty c; + if ( + (a = lambda_param_rule(p)) // lambda_param + && + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (_literal_1 = _PyPegen_expect_token(p, 21)) // token='>' + && + (c = expression_rule(p)) // expression + && + _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 11) // token=':' + ) + { + D(fprintf(stderr, "%*c+ lambda_param_maybe_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param '=' '>' expression &':'")); + _res = _PyPegen_late_default_pair ( p , a , c , NULL ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s lambda_param_maybe_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param '=' '>' expression &':'")); + } { // lambda_param default? ',' if (p->error_indicator) { p->level--; @@ -16162,7 +16472,7 @@ lambda_param_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_arg ( a -> v . Name . id , NULL , NULL , EXTRA ); + _res = _PyAST_arg ( a -> v . Name . id , NULL , NULL , NULL , EXTRA ); if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; diff --git a/Parser/pegen.h b/Parser/pegen.h index 7454289ef6df03d..4cb7a0155758c05 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -368,6 +368,7 @@ KeyPatternPair *_PyPegen_key_pattern_pair(Parser *, expr_ty, pattern_ty); asdl_expr_seq *_PyPegen_get_pattern_keys(Parser *, asdl_seq *); asdl_pattern_seq *_PyPegen_get_patterns(Parser *, asdl_seq *); NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty, Token *); +NameDefaultPair *_PyPegen_late_default_pair(Parser *, arg_ty, expr_ty, Token *); SlashWithDefault *_PyPegen_slash_with_default(Parser *, asdl_arg_seq *, asdl_seq *); StarEtc *_PyPegen_star_etc(Parser *, arg_ty, asdl_seq *, arg_ty); arguments_ty _PyPegen_make_arguments(Parser *, asdl_arg_seq *, SlashWithDefault *, diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 0d89f7e87f1fa2d..0975bec3d35a4bd 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -237,6 +237,7 @@ void _PyAST_Fini(PyInterpreterState *interp) offsetof(struct ast_state, kwd_attrs), offsetof(struct ast_state, kwd_patterns), offsetof(struct ast_state, kwonlyargs), + offsetof(struct ast_state, late_default), offsetof(struct ast_state, left), offsetof(struct ast_state, level), offsetof(struct ast_state, lineno), @@ -347,6 +348,7 @@ static int init_identifiers(struct ast_state *state) if ((state->kwd_attrs = PyUnicode_InternFromString("kwd_attrs")) == NULL) return -1; if ((state->kwd_patterns = PyUnicode_InternFromString("kwd_patterns")) == NULL) return -1; if ((state->kwonlyargs = PyUnicode_InternFromString("kwonlyargs")) == NULL) return -1; + if ((state->late_default = PyUnicode_InternFromString("late_default")) == NULL) return -1; if ((state->left = PyUnicode_InternFromString("left")) == NULL) return -1; if ((state->level = PyUnicode_InternFromString("level")) == NULL) return -1; if ((state->lineno = PyUnicode_InternFromString("lineno")) == NULL) return -1; @@ -726,6 +728,7 @@ static const char * const arg_fields[]={ "arg", "annotation", "type_comment", + "late_default", }; static PyObject* ast2obj_keyword(struct ast_state *state, void*); static const char * const keyword_attributes[] = { @@ -1177,6 +1180,8 @@ add_ast_annotations(struct ast_state *state) offsetof(struct ast_state, expr_type), FIELD_OPTIONAL}, {offsetof(struct ast_state, type_comment), 0, FIELD_OPTIONAL | FIELD_BUILTIN}, + {offsetof(struct ast_state, late_default), + 0, FIELD_OPTIONAL | FIELD_BUILTIN}, {offsetof(struct ast_state, arg), 0, FIELD_OPTIONAL | FIELD_BUILTIN}, {offsetof(struct ast_state, value), @@ -1343,23 +1348,23 @@ add_ast_annotations(struct ast_state *state) {offsetof(struct ast_state, comprehension_type), 149, 4}, {offsetof(struct ast_state, ExceptHandler_type), 153, 3}, {offsetof(struct ast_state, arguments_type), 156, 7}, - {offsetof(struct ast_state, arg_type), 163, 3}, - {offsetof(struct ast_state, keyword_type), 166, 2}, - {offsetof(struct ast_state, alias_type), 168, 2}, - {offsetof(struct ast_state, withitem_type), 170, 2}, - {offsetof(struct ast_state, match_case_type), 172, 3}, - {offsetof(struct ast_state, MatchValue_type), 175, 1}, - {offsetof(struct ast_state, MatchSingleton_type), 176, 1}, - {offsetof(struct ast_state, MatchSequence_type), 177, 1}, - {offsetof(struct ast_state, MatchMapping_type), 178, 3}, - {offsetof(struct ast_state, MatchClass_type), 181, 4}, - {offsetof(struct ast_state, MatchStar_type), 185, 1}, - {offsetof(struct ast_state, MatchAs_type), 186, 2}, - {offsetof(struct ast_state, MatchOr_type), 188, 1}, - {offsetof(struct ast_state, TypeIgnore_type), 189, 2}, - {offsetof(struct ast_state, TypeVar_type), 191, 3}, - {offsetof(struct ast_state, ParamSpec_type), 194, 2}, - {offsetof(struct ast_state, TypeVarTuple_type), 196, 2}, + {offsetof(struct ast_state, arg_type), 163, 4}, + {offsetof(struct ast_state, keyword_type), 167, 2}, + {offsetof(struct ast_state, alias_type), 169, 2}, + {offsetof(struct ast_state, withitem_type), 171, 2}, + {offsetof(struct ast_state, match_case_type), 173, 3}, + {offsetof(struct ast_state, MatchValue_type), 176, 1}, + {offsetof(struct ast_state, MatchSingleton_type), 177, 1}, + {offsetof(struct ast_state, MatchSequence_type), 178, 1}, + {offsetof(struct ast_state, MatchMapping_type), 179, 3}, + {offsetof(struct ast_state, MatchClass_type), 182, 4}, + {offsetof(struct ast_state, MatchStar_type), 186, 1}, + {offsetof(struct ast_state, MatchAs_type), 187, 2}, + {offsetof(struct ast_state, MatchOr_type), 189, 1}, + {offsetof(struct ast_state, TypeIgnore_type), 190, 2}, + {offsetof(struct ast_state, TypeVar_type), 192, 3}, + {offsetof(struct ast_state, ParamSpec_type), 195, 2}, + {offsetof(struct ast_state, TypeVarTuple_type), 197, 2}, }; char *base = (char *)state; PyObject *annotations = NULL; @@ -3040,8 +3045,8 @@ init_types(void *arg) return -1; if (PyObject_SetAttr(state->arguments_type, state->kwarg, Py_None) == -1) return -1; - state->arg_type = make_type(state, "arg", state->AST_type, arg_fields, 3, - "arg(identifier arg, expr? annotation, string? type_comment)"); + state->arg_type = make_type(state, "arg", state->AST_type, arg_fields, 4, + "arg(identifier arg, expr? annotation, string? type_comment, string? late_default)"); if (!state->arg_type) return -1; if (add_attributes(state, state->arg_type, arg_attributes, 4) < 0) return -1; @@ -3049,6 +3054,8 @@ init_types(void *arg) return -1; if (PyObject_SetAttr(state->arg_type, state->type_comment, Py_None) == -1) return -1; + if (PyObject_SetAttr(state->arg_type, state->late_default, Py_None) == -1) + return -1; if (PyObject_SetAttr(state->arg_type, state->end_lineno, Py_None) == -1) return -1; if (PyObject_SetAttr(state->arg_type, state->end_col_offset, Py_None) == -1) @@ -4761,8 +4768,9 @@ _PyAST_arguments(asdl_arg_seq * posonlyargs, asdl_arg_seq * args, arg_ty } arg_ty -_PyAST_arg(identifier arg, expr_ty annotation, string type_comment, int lineno, - int col_offset, int end_lineno, int end_col_offset, PyArena *arena) +_PyAST_arg(identifier arg, expr_ty annotation, string type_comment, string + late_default, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena) { arg_ty p; if (!arg) { @@ -4776,6 +4784,7 @@ _PyAST_arg(identifier arg, expr_ty annotation, string type_comment, int lineno, p->arg = arg; p->annotation = annotation; p->type_comment = type_comment; + p->late_default = late_default; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -6613,6 +6622,11 @@ ast2obj_arg(struct ast_state *state, void* _o) if (PyObject_SetAttr(result, state->type_comment, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(state, o->late_default); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->late_default, value) == -1) + goto failed; + Py_DECREF(value); value = ast2obj_int(state, o->lineno); if (!value) goto failed; if (PyObject_SetAttr(result, state->lineno, value) < 0) @@ -12870,6 +12884,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, const char* identifier arg; expr_ty annotation; string type_comment; + string late_default; int lineno; int col_offset; int end_lineno; @@ -12926,6 +12941,23 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, const char* if (res != 0) goto failed; Py_CLEAR(tmp); } + if (PyObject_GetOptionalAttr(obj, state->late_default, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + late_default = NULL; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'arg' node")) { + goto failed; + } + res = obj2ast_string(state, tmp, &late_default, "late_default", arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { return -1; } @@ -12994,8 +13026,8 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, const char* if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_arg(arg, annotation, type_comment, lineno, col_offset, - end_lineno, end_col_offset, arena); + *out = _PyAST_arg(arg, annotation, type_comment, late_default, lineno, + col_offset, end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; failed: diff --git a/Python/ast.c b/Python/ast.c index 1befd1778114927..7218efe62106898 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -150,6 +150,29 @@ validate_arguments(arguments_ty args) "kw_defaults on arguments"); return 0; } + Py_ssize_t nposonly = asdl_seq_LEN(args->posonlyargs); + Py_ssize_t npos = nposonly + asdl_seq_LEN(args->args); + Py_ssize_t first_default = npos - asdl_seq_LEN(args->defaults); + for (Py_ssize_t i = 0; i < npos; i++) { + arg_ty arg = i < nposonly ? asdl_seq_GET(args->posonlyargs, i) + : asdl_seq_GET(args->args, i - nposonly); + if (arg->late_default && i < first_default) { + PyErr_SetString(PyExc_ValueError, "late-bound parameter has no default"); + return 0; + } + } + for (Py_ssize_t i = 0; i < asdl_seq_LEN(args->kwonlyargs); i++) { + if (asdl_seq_GET(args->kwonlyargs, i)->late_default + && asdl_seq_GET(args->kw_defaults, i) == NULL) { + PyErr_SetString(PyExc_ValueError, "late-bound parameter has no default"); + return 0; + } + } + if ((args->vararg && args->vararg->late_default) + || (args->kwarg && args->kwarg->late_default)) { + PyErr_SetString(PyExc_ValueError, "variadic parameter cannot have a late default"); + return 0; + } return validate_exprs(args->defaults, Load, 0) && validate_exprs(args->kw_defaults, Load, 1); } diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index 6050c351cff68f0..ac2c8344437b84b 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -252,7 +252,9 @@ append_ast_args(PyUnicodeWriter *writer, arguments_ty args) di = i - posonlyarg_count - arg_count + default_count; if (di >= 0) { - APPEND_CHAR('='); + arg_ty arg = i < posonlyarg_count ? asdl_seq_GET(args->posonlyargs, i) + : asdl_seq_GET(args->args, i - posonlyarg_count); + APPEND_STR(arg->late_default ? "=>" : "="); APPEND_EXPR((expr_ty)asdl_seq_GET(args->defaults, di), PR_TEST); } if (posonlyarg_count && i + 1 == posonlyarg_count) { @@ -280,7 +282,8 @@ append_ast_args(PyUnicodeWriter *writer, arguments_ty args) if (di >= 0) { expr_ty default_ = (expr_ty)asdl_seq_GET(args->kw_defaults, di); if (default_) { - APPEND_CHAR('='); + arg_ty arg = asdl_seq_GET(args->kwonlyargs, i); + APPEND_STR(arg->late_default ? "=>" : "="); APPEND_EXPR(default_, PR_TEST); } } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 31eaeab0d67841f..55998c22986ca68 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -270,6 +270,18 @@ dummy_func( LOAD_FAST, }; + pure inst(QUERY_FAST, (-- value)) { + value = PyStackRef_IsNull(GETLOCAL(oparg)) + ? PyStackRef_False : PyStackRef_True; + } + + inst(QUERY_DEREF, (-- value)) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *contents = PyCell_GetRef(cell); + value = contents == NULL ? PyStackRef_False : PyStackRef_True; + Py_XDECREF(contents); + } + inst(LOAD_FAST_CHECK, (-- value)) { _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { diff --git a/Python/ceval.c b/Python/ceval.c index 43dc48f375301cc..1aca2fcf367a9e0 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -412,6 +412,8 @@ const size_t _Py_FunctionAttributeOffsets[] = { [MAKE_FUNCTION_KWDEFAULTS] = offsetof(PyFunctionObject, func_kwdefaults), [MAKE_FUNCTION_DEFAULTS] = offsetof(PyFunctionObject, func_defaults), [MAKE_FUNCTION_ANNOTATE] = offsetof(PyFunctionObject, func_annotate), + [MAKE_FUNCTION_DEFAULTS_EXTRA] = offsetof(PyFunctionObject, func_defaults_extra), + [MAKE_FUNCTION_KWDEFAULTS_EXTRA] = offsetof(PyFunctionObject, func_kwdefaults_extra), }; // PEP 634: Structural Pattern Matching @@ -1973,6 +1975,12 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, for (; i < defcount; i++) { if (PyStackRef_AsPyObjectBorrow(localsplus[m+i]) == NULL) { PyObject *def = defs[i]; + if ((co->co_flags & CO_LATE_DEFAULTS) && def == Py_Ellipsis + && func->func_defaults_extra != NULL + && i < PyTuple_GET_SIZE(func->func_defaults_extra) + && PyTuple_GET_ITEM(func->func_defaults_extra, i) != Py_None) { + continue; + } localsplus[m+i] = PyStackRef_FromPyObjectNew(def); } } @@ -1981,7 +1989,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, /* Add missing keyword arguments (copy default values from kwdefs) */ if (co->co_kwonlyargcount > 0) { - Py_ssize_t missing = 0; + PyObject *missing_names = NULL; for (i = co->co_argcount; i < total_args; i++) { if (PyStackRef_AsPyObjectBorrow(localsplus[i]) != NULL) continue; @@ -1989,18 +1997,54 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (func->func_kwdefaults != NULL) { PyObject *def; if (PyDict_GetItemRef(func->func_kwdefaults, varname, &def) < 0) { + Py_XDECREF(missing_names); goto fail_post_args; } if (def) { + if ((co->co_flags & CO_LATE_DEFAULTS) && def == Py_Ellipsis + && func->func_kwdefaults_extra != NULL) { + PyObject *extra; + if (PyDict_GetItemRef(func->func_kwdefaults_extra, varname, &extra) < 0) { + Py_DECREF(def); + Py_XDECREF(missing_names); + goto fail_post_args; + } + if (extra != NULL && extra != Py_None) { + Py_DECREF(extra); + Py_DECREF(def); + continue; + } + Py_XDECREF(extra); + } localsplus[i] = PyStackRef_FromPyObjectSteal(def); continue; } } - missing++; + /* Record required names while binding. Re-reading a mutable + defaults dict later could disagree with which arguments were + omitted, and late defaults intentionally leave a NULL local. */ + if (missing_names == NULL) { + missing_names = PyList_New(0); + if (missing_names == NULL) { + goto fail_post_args; + } + } + PyObject *name = PyObject_Repr(varname); + if (name == NULL) { + Py_DECREF(missing_names); + goto fail_post_args; + } + int err = PyList_Append(missing_names, name); + Py_DECREF(name); + if (err < 0) { + Py_DECREF(missing_names); + goto fail_post_args; + } } - if (missing) { - missing_arguments(tstate, co, missing, -1, localsplus, - func->func_qualname); + if (missing_names != NULL) { + format_missing(tstate, "keyword-only", co, missing_names, + func->func_qualname); + Py_DECREF(missing_names); goto fail_post_args; } } diff --git a/Python/codegen.c b/Python/codegen.c index 840dd3103a81c5a..15741c24b2db3d8 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -1025,9 +1025,15 @@ codegen_make_closure(compiler *c, location loc, if (flags & MAKE_FUNCTION_ANNOTATE) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE); } + if (flags & MAKE_FUNCTION_KWDEFAULTS_EXTRA) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS_EXTRA); + } if (flags & MAKE_FUNCTION_KWDEFAULTS) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS); } + if (flags & MAKE_FUNCTION_DEFAULTS_EXTRA) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_DEFAULTS_EXTRA); + } if (flags & MAKE_FUNCTION_DEFAULTS) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_DEFAULTS); } @@ -1061,6 +1067,68 @@ codegen_apply_decorators(compiler *c, asdl_expr_seq* decos) return SUCCESS; } +/* Return the positional parameter corresponding to a defaults entry. */ +static arg_ty +codegen_default_arg(arguments_ty args, Py_ssize_t index) +{ + Py_ssize_t nposonly = asdl_seq_LEN(args->posonlyargs); + Py_ssize_t offset = nposonly + asdl_seq_LEN(args->args) + - asdl_seq_LEN(args->defaults) + index; + return offset < nposonly ? asdl_seq_GET(args->posonlyargs, offset) + : asdl_seq_GET(args->args, offset - nposonly); +} + +static int +codegen_late_default(compiler *c, arg_ty arg, expr_ty value) +{ + if (!arg->late_default) { + return SUCCESS; + } + location loc = LOC(value); + PyObject *mangled = _PyCompile_MaybeMangle(c, arg->arg); + if (mangled == NULL) { + return ERROR; + } + _PyCompile_optype optype; + Py_ssize_t index = 0; + int scope = _PyCompile_ResolveNameop(c, mangled, &optype, &index); + if (scope < 0) { + Py_DECREF(mangled); + return ERROR; + } + if (optype == COMPILE_OP_DEREF) { + Py_DECREF(mangled); + ADDOP_I(c, loc, QUERY_DEREF, index); + } + else { + assert(optype == COMPILE_OP_FAST); + ADDOP_N(c, loc, QUERY_FAST, mangled, varnames); + } + NEW_JUMP_TARGET_LABEL(c, bound); + ADDOP_JUMP(c, loc, POP_JUMP_IF_TRUE, bound); + VISIT(c, expr, value); + RETURN_IF_ERROR(codegen_nameop(c, loc, arg->arg, Store)); + USE_LABEL(c, bound); + return SUCCESS; +} + +static int +codegen_late_defaults(compiler *c, arguments_ty args) +{ + for (Py_ssize_t i = 0; i < asdl_seq_LEN(args->defaults); i++) { + RETURN_IF_ERROR(codegen_late_default(c, codegen_default_arg(args, i), + asdl_seq_GET(args->defaults, i))); + } + for (Py_ssize_t i = 0; i < asdl_seq_LEN(args->kwonlyargs); i++) { + arg_ty arg = asdl_seq_GET(args->kwonlyargs, i); + if (arg->late_default) { + RETURN_IF_ERROR(codegen_late_default(c, arg, + asdl_seq_GET(args->kw_defaults, i))); + } + } + return SUCCESS; +} + static int codegen_kwonlydefaults(compiler *c, location loc, asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults) @@ -1080,7 +1148,12 @@ codegen_kwonlydefaults(compiler *c, location loc, return ERROR; } ADDOP_LOAD_CONST_NEW(c, loc, mangled); - VISIT(c, expr, default_); + if (arg->late_default) { + ADDOP_LOAD_CONST(c, loc, Py_Ellipsis); + } + else { + VISIT(c, expr, default_); + } } } if (default_count) { @@ -1221,7 +1294,14 @@ static int codegen_defaults(compiler *c, arguments_ty args, location loc) { - VISIT_SEQ(c, expr, args->defaults); + for (Py_ssize_t i = 0; i < asdl_seq_LEN(args->defaults); i++) { + if (codegen_default_arg(args, i)->late_default) { + ADDOP_LOAD_CONST(c, loc, Py_Ellipsis); + } + else { + VISIT(c, expr, asdl_seq_GET(args->defaults, i)); + } + } ADDOP_I(c, loc, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); return SUCCESS; } @@ -1234,6 +1314,23 @@ codegen_default_arguments(compiler *c, location loc, if (args->defaults && asdl_seq_LEN(args->defaults) > 0) { RETURN_IF_ERROR(codegen_defaults(c, args, loc)); funcflags |= MAKE_FUNCTION_DEFAULTS; + int has_late = 0; + for (Py_ssize_t i = 0; i < asdl_seq_LEN(args->defaults); i++) { + has_late |= codegen_default_arg(args, i)->late_default != NULL; + } + if (has_late) { + for (Py_ssize_t i = 0; i < asdl_seq_LEN(args->defaults); i++) { + if (codegen_default_arg(args, i)->late_default) { + ADDOP_LOAD_CONST_NEW(c, loc, + Py_NewRef(codegen_default_arg(args, i)->late_default)); + } + else { + ADDOP_LOAD_CONST(c, loc, Py_None); + } + } + ADDOP_I(c, loc, BUILD_TUPLE, asdl_seq_LEN(args->defaults)); + funcflags |= MAKE_FUNCTION_DEFAULTS_EXTRA; + } } if (args->kwonlyargs) { int res = codegen_kwonlydefaults(c, loc, @@ -1242,6 +1339,20 @@ codegen_default_arguments(compiler *c, location loc, RETURN_IF_ERROR(res); if (res > 0) { funcflags |= MAKE_FUNCTION_KWDEFAULTS; + int count = 0; + for (Py_ssize_t i = 0; i < asdl_seq_LEN(args->kwonlyargs); i++) { + arg_ty arg = asdl_seq_GET(args->kwonlyargs, i); + if (arg->late_default) { + ADDOP_LOAD_CONST_NEW(c, loc, _PyCompile_MaybeMangle(c, arg->arg)); + ADDOP_LOAD_CONST_NEW(c, loc, + Py_NewRef(arg->late_default)); + count++; + } + } + if (count) { + ADDOP_I(c, loc, BUILD_MAP, count); + funcflags |= MAKE_FUNCTION_KWDEFAULTS_EXTRA; + } } } return funcflags; @@ -1397,7 +1508,7 @@ codegen_type_params(compiler *c, asdl_type_param_seq *type_params) } static int -codegen_emit_function_body(compiler *c, asdl_stmt_seq *body) +codegen_emit_function_body(compiler *c, asdl_stmt_seq *body, arguments_ty args) { PySTEntryObject *ste = SYMTABLE_ENTRY(c); Py_ssize_t first_instr = 0; @@ -1424,6 +1535,7 @@ codegen_emit_function_body(compiler *c, asdl_stmt_seq *body) start, NO_LABEL, NULL)); } + RETURN_IF_ERROR(codegen_late_defaults(c, args)); for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(body); i++) { VISIT(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); } @@ -1469,7 +1581,7 @@ codegen_function_body(compiler *c, stmt_ty s, int is_async, Py_ssize_t funcflags RETURN_IF_ERROR( codegen_enter_scope(c, name, scope_type, (void *)s, firstlineno, NULL, &umd)); - RETURN_IF_ERROR_IN_SCOPE(c, codegen_emit_function_body(c, body)); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_emit_function_body(c, body, args)); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1); _PyCompile_ExitScope(c); @@ -1533,8 +1645,14 @@ codegen_function(compiler *c, stmt_ty s, int is_async) if (funcflags & MAKE_FUNCTION_KWDEFAULTS) { num_typeparam_args += 1; } - if (num_typeparam_args == 2) { - ADDOP_I(c, loc, SWAP, 2); + if (funcflags & MAKE_FUNCTION_DEFAULTS_EXTRA) { + num_typeparam_args++; + } + if (funcflags & MAKE_FUNCTION_KWDEFAULTS_EXTRA) { + num_typeparam_args++; + } + for (int i = num_typeparam_args; i > 1; i--) { + ADDOP_I(c, loc, SWAP, i); } PyObject *type_params_name = PyUnicode_FromFormat("", name); if (!type_params_name) { @@ -2112,6 +2230,7 @@ codegen_lambda(compiler *c, expr_ty e) assert(!SYMTABLE_ENTRY(c)->ste_has_docstring); + RETURN_IF_ERROR_IN_SCOPE(c, codegen_late_defaults(c, args)); VISIT_IN_SCOPE(c, expr, e->v.Lambda.body); if (SYMTABLE_ENTRY(c)->ste_generator) { co = _PyCompile_OptimizeAndAssemble(c, 0); diff --git a/Python/compile.c b/Python/compile.c index b574767a9180b1a..31d49b1834d14a2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1600,6 +1600,8 @@ compute_code_flags(compiler *c) flags |= CO_HAS_DOCSTRING; if (ste->ste_method) flags |= CO_METHOD; + if (ste->ste_has_late_defaults) + flags |= CO_LATE_DEFAULTS; } if (ste->ste_coroutine && !ste->ste_generator) { @@ -1630,7 +1632,9 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, goto error; } int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); - int nparams = (int)PyList_GET_SIZE(u->u_ste->ste_varnames); + /* Late-bound parameters can still be unbound in the function prologue. */ + int nparams = u->u_ste->ste_has_late_defaults ? 0 + : (int)PyList_GET_SIZE(u->u_ste->ste_varnames); assert(u->u_metadata.u_firstlineno); if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c5b2dfcf5f618f3..76c5c2e47267b9a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -257,6 +257,76 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + case _QUERY_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + oparg = CURRENT_OPARG(); + value = PyStackRef_IsNull(GETLOCAL(oparg)) + ? PyStackRef_False : PyStackRef_True; + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _QUERY_FAST_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + value = PyStackRef_IsNull(GETLOCAL(oparg)) + ? PyStackRef_False : PyStackRef_True; + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _QUERY_FAST_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + value = PyStackRef_IsNull(GETLOCAL(oparg)) + ? PyStackRef_False : PyStackRef_True; + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _QUERY_DEREF_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + oparg = CURRENT_OPARG(); + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *contents = PyCell_GetRef(cell); + value = contents == NULL ? PyStackRef_False : PyStackRef_True; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_XDECREF(contents); + _PyFrame_StackPointerInvalidate(frame); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _LOAD_FAST_CHECK_r01: { CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/flowgraph.c b/Python/flowgraph.c index a5138d1a1fa2846..85e31a0f8892c01 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -3976,6 +3976,7 @@ fix_cell_offsets(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int * case MAKE_CELL: case LOAD_CLOSURE: case LOAD_DEREF: + case QUERY_DEREF: case STORE_DEREF: case DELETE_DEREF: case LOAD_FROM_DICT_OR_DEREF: diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dd0ce41e4b06b4a..45662112b5ba219 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -11142,6 +11142,45 @@ DISPATCH(); } + TARGET(QUERY_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = QUERY_DEREF; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(QUERY_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *contents = PyCell_GetRef(cell); + value = contents == NULL ? PyStackRef_False : PyStackRef_True; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_XDECREF(contents); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(QUERY_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = QUERY_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(QUERY_FAST); + _PyStackRef value; + value = PyStackRef_IsNull(GETLOCAL(oparg)) + ? PyStackRef_False : PyStackRef_True; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + TARGET(RAISE_VARARGS) { #if _Py_TAIL_CALL_INTERP int opcode = RAISE_VARARGS; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4b..4d3d67d8457d687 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -100,6 +100,8 @@ static void *opcode_targets_table[256] = { &&TARGET_POP_JUMP_IF_NONE, &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_TRUE, + &&TARGET_QUERY_DEREF, + &&TARGET_QUERY_FAST, &&TARGET_RAISE_VARARGS, &&TARGET_RERAISE, &&TARGET_SEND, @@ -126,8 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -376,8 +376,8 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -713,6 +713,8 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAM static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_QUERY_DEREF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_QUERY_FAST(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); @@ -956,6 +958,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [POP_TOP] = _TAIL_CALL_POP_TOP, [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, + [QUERY_DEREF] = _TAIL_CALL_QUERY_DEREF, + [QUERY_FAST] = _TAIL_CALL_QUERY_FAST, [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, [RERAISE] = _TAIL_CALL_RERAISE, [RESERVED] = _TAIL_CALL_RESERVED, @@ -1005,8 +1009,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1214,6 +1216,8 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [POP_TOP] = _TAIL_CALL_TRACE_RECORD, [PUSH_EXC_INFO] = _TAIL_CALL_TRACE_RECORD, [PUSH_NULL] = _TAIL_CALL_TRACE_RECORD, + [QUERY_DEREF] = _TAIL_CALL_TRACE_RECORD, + [QUERY_FAST] = _TAIL_CALL_TRACE_RECORD, [RAISE_VARARGS] = _TAIL_CALL_TRACE_RECORD, [RERAISE] = _TAIL_CALL_TRACE_RECORD, [RESERVED] = _TAIL_CALL_TRACE_RECORD, @@ -1263,8 +1267,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9cd20c495db4418..b0f40d1d06bb4bf 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -25,6 +25,26 @@ /* _MONITOR_RESUME is not a viable micro-op for tier 2 */ + case _QUERY_FAST: { + JitOptRef value; + value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _QUERY_DEREF: { + JitOptRef value; + value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + case _LOAD_FAST_CHECK: { JitOptRef value; value = GETLOCAL(oparg); diff --git a/Python/symtable.c b/Python/symtable.c index b97778ada20c04b..68c3e80bc9dd6d5 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -154,6 +154,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_in_unevaluated_annotation = 0; ste->ste_annotation_block = NULL; + ste->ste_has_late_defaults = 0; ste->ste_has_docstring = 0; ste->ste_method = 0; @@ -272,6 +273,7 @@ static int symtable_visit_listcomp(struct symtable *st, expr_ty s); static int symtable_visit_setcomp(struct symtable *st, expr_ty s); static int symtable_visit_dictcomp(struct symtable *st, expr_ty s); static int symtable_visit_arguments(struct symtable *st, arguments_ty); +static int symtable_visit_defaults(struct symtable *st, arguments_ty, int); static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); @@ -1681,7 +1683,8 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag, static int symtable_enter_type_param_block(struct symtable *st, identifier name, void *ast, int has_defaults, int has_kwdefaults, - enum _stmt_kind kind, _Py_SourceLocation loc) + arguments_ty defaults_args, enum _stmt_kind kind, + _Py_SourceLocation loc) { _Py_block_ty current_type = st->st_cur->ste_type; if(!symtable_enter_block(st, name, TypeParametersBlock, ast, loc)) { @@ -1718,12 +1721,39 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, goto error; } } + if (defaults_args != NULL) { + int has_late = 0; + for (Py_ssize_t i = 0; i < asdl_seq_LEN(defaults_args->posonlyargs); i++) { + has_late |= asdl_seq_GET(defaults_args->posonlyargs, i)->late_default != NULL; + } + for (Py_ssize_t i = 0; i < asdl_seq_LEN(defaults_args->args); i++) { + has_late |= asdl_seq_GET(defaults_args->args, i)->late_default != NULL; + } + if (has_late) { + _Py_DECLARE_STR(defaults_extra, ".defaults_extra"); + if (!symtable_add_def(st, &_Py_STR(defaults_extra), DEF_PARAM, loc)) { + goto error; + } + } + } if (has_kwdefaults) { _Py_DECLARE_STR(kwdefaults, ".kwdefaults"); if (!symtable_add_def(st, &_Py_STR(kwdefaults), DEF_PARAM, loc)) { goto error; } } + if (defaults_args != NULL) { + int has_late = 0; + for (Py_ssize_t i = 0; i < asdl_seq_LEN(defaults_args->kwonlyargs); i++) { + has_late |= asdl_seq_GET(defaults_args->kwonlyargs, i)->late_default != NULL; + } + if (has_late) { + _Py_DECLARE_STR(kwdefaults_extra, ".kwdefaults_extra"); + if (!symtable_add_def(st, &_Py_STR(kwdefaults_extra), DEF_PARAM, loc)) { + goto error; + } + } + } return 1; error: symtable_exit_block(st); @@ -1916,10 +1946,9 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) case FunctionDef_kind: { if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s))) return 0; - if (s->v.FunctionDef.args->defaults) - VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults); - if (s->v.FunctionDef.args->kw_defaults) - VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults); + if (!symtable_visit_defaults(st, s->v.FunctionDef.args, 0)) { + return 0; + } if (s->v.FunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list); if (asdl_seq_LEN(s->v.FunctionDef.type_params) > 0) { @@ -1929,7 +1958,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) s->v.FunctionDef.args->defaults != NULL, has_kwonlydefaults(s->v.FunctionDef.args->kwonlyargs, s->v.FunctionDef.args->kw_defaults), - s->kind, + s->v.FunctionDef.args, s->kind, LOCATION(s))) { return 0; } @@ -1975,7 +2004,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) { if (!symtable_enter_type_param_block(st, s->v.ClassDef.name, (void *)s->v.ClassDef.type_params, - false, false, s->kind, + false, false, NULL, s->kind, LOCATION(s))) { return 0; } @@ -2032,7 +2061,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) if (!symtable_enter_type_param_block( st, name, (void *)s->v.TypeAlias.type_params, - false, false, s->kind, + false, false, NULL, s->kind, LOCATION(s))) { return 0; } @@ -2300,11 +2329,9 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) case AsyncFunctionDef_kind: { if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s))) return 0; - if (s->v.AsyncFunctionDef.args->defaults) - VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults); - if (s->v.AsyncFunctionDef.args->kw_defaults) - VISIT_SEQ_WITH_NULL(st, expr, - s->v.AsyncFunctionDef.args->kw_defaults); + if (!symtable_visit_defaults(st, s->v.AsyncFunctionDef.args, 0)) { + return 0; + } if (s->v.AsyncFunctionDef.decorator_list) VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list); if (asdl_seq_LEN(s->v.AsyncFunctionDef.type_params) > 0) { @@ -2314,7 +2341,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) s->v.AsyncFunctionDef.args->defaults != NULL, has_kwonlydefaults(s->v.AsyncFunctionDef.args->kwonlyargs, s->v.AsyncFunctionDef.args->kw_defaults), - s->kind, + s->v.AsyncFunctionDef.args, s->kind, LOCATION(s))) { return 0; } @@ -2521,10 +2548,9 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT(st, expr, e->v.UnaryOp.operand); break; case Lambda_kind: { - if (e->v.Lambda.args->defaults) - VISIT_SEQ(st, expr, e->v.Lambda.args->defaults); - if (e->v.Lambda.args->kw_defaults) - VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults); + if (!symtable_visit_defaults(st, e->v.Lambda.args, 0)) { + return 0; + } if (!symtable_enter_block(st, &_Py_STR(anon_lambda), FunctionBlock, (void *)e, LOCATION(e))) { return 0; @@ -2977,6 +3003,35 @@ symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ return 1; } +static int +symtable_visit_defaults(struct symtable *st, arguments_ty a, int late) +{ + Py_ssize_t nposonly = asdl_seq_LEN(a->posonlyargs); + Py_ssize_t npos = nposonly + asdl_seq_LEN(a->args); + Py_ssize_t first = npos - asdl_seq_LEN(a->defaults); + for (Py_ssize_t i = first; i < npos; i++) { + arg_ty arg = i < nposonly ? asdl_seq_GET(a->posonlyargs, i) + : asdl_seq_GET(a->args, i - nposonly); + if (!!arg->late_default == late) { + VISIT(st, expr, asdl_seq_GET(a->defaults, i - first)); + if (late) { + st->st_cur->ste_has_late_defaults = 1; + } + } + } + for (Py_ssize_t i = 0; i < asdl_seq_LEN(a->kwonlyargs); i++) { + arg_ty arg = asdl_seq_GET(a->kwonlyargs, i); + expr_ty value = asdl_seq_GET(a->kw_defaults, i); + if (value && !!arg->late_default == late) { + VISIT(st, expr, value); + if (late) { + st->st_cur->ste_has_late_defaults = 1; + } + } + } + return 1; +} + static int symtable_visit_arguments(struct symtable *st, arguments_ty a) { @@ -2999,7 +3054,7 @@ symtable_visit_arguments(struct symtable *st, arguments_ty a) return 0; st->st_cur->ste_varkeywords = 1; } - return 1; + return symtable_visit_defaults(st, a, 1); }