diff --git a/Doc/builtins/functions.rst b/Doc/builtins/functions.rst index f6203b026d32a4f..6cc9e7c6b8faaf8 100644 --- a/Doc/builtins/functions.rst +++ b/Doc/builtins/functions.rst @@ -1254,8 +1254,9 @@ are always available. They are listed here in alphabetical order. .. impl-detail:: - ``len`` raises :exc:`OverflowError` on lengths larger than - :data:`sys.maxsize`, such as :class:`range(2 ** 100) `. + CPython's C length protocol is limited to :data:`sys.maxsize`. + When that limit is reached, ``len`` may still return a larger Python + integer if the object's :meth:`~object.__len__` method can provide one. .. _func-list: diff --git a/Doc/builtins/stdtypes.rst b/Doc/builtins/stdtypes.rst index cf05f947c5908df..903a68d3f0be4bb 100644 --- a/Doc/builtins/stdtypes.rst +++ b/Doc/builtins/stdtypes.rst @@ -1815,8 +1815,9 @@ loops. indices. Ranges containing absolute values larger than :data:`sys.maxsize` are - permitted but some features (such as :func:`len`) may raise - :exc:`OverflowError`. + permitted, though some operations that use the C length protocol may still + raise :exc:`OverflowError` for ranges with lengths larger than + :data:`!sys.maxsize`. Range examples:: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index c860a3356628af1..50a93b946028568 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -3245,11 +3245,10 @@ through the object's keys; for sequences, it should iterate through the values. .. impl-detail:: - In CPython, the length is required to be at most :data:`sys.maxsize`. - If the length is larger than :data:`!sys.maxsize` some features (such as - :func:`len`) may raise :exc:`OverflowError`. To prevent raising - :exc:`!OverflowError` by truth value testing, an object must define a - :meth:`~object.__bool__` method. + In CPython, the C length protocol is limited to :data:`sys.maxsize`. + Some features that use that protocol may raise :exc:`OverflowError` + for larger lengths. To prevent raising :exc:`!OverflowError` by truth + value testing, an object must define a :meth:`~object.__bool__` method. .. method:: object.__length_hint__(self) diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index 67c6fa7c0c4ed5b..97609bdc7b79bea 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -21,6 +21,7 @@ PyAPI_FUNC(PyObject *) _PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); PyAPI_FUNC(PyObject *) _PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o); +PyAPI_FUNC(PyObject *) _PyObject_LengthAsPyLong(PyObject *o); /* === Sequence protocol ================================================ */ diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 294355698c8683b..9f9ed8d74e0cff2 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1385,9 +1385,18 @@ def __len__(self): return -10 self.assertRaises(ValueError, len, NegativeLen()) class HugeLen: + def __init__(self): + self.calls = 0 + def __len__(self): + self.calls += 1 return sys.maxsize + 1 - self.assertRaises(OverflowError, len, HugeLen()) + huge_len = HugeLen() + self.assertEqual(len(huge_len), sys.maxsize + 1) + self.assertEqual(huge_len.calls, 1) + huge_len.__len__ = lambda: 0 + self.assertEqual(len(huge_len), sys.maxsize + 1) + self.assertEqual(huge_len.calls, 2) class HugeNegativeLen: def __len__(self): return -sys.maxsize-10 diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 8d093ab1b7014a4..08455233cc25cf4 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -5,6 +5,7 @@ import time import pickle import shlex +import sys import warnings import test.support @@ -122,6 +123,7 @@ def test_choice(self): choice([]) self.assertEqual(choice([50]), 50) self.assertIn(choice([25, 75]), [25, 75]) + self.assertIn(choice(range(sys.maxsize * 2)), range(sys.maxsize * 2)) def test_choice_with_numpy(self): # Accommodation for NumPy arrays which have disabled __bool__(). @@ -177,6 +179,11 @@ def test_sample_inputs(self): self.gen.sample(range(20), 2) self.gen.sample(str('abcdefghijklmnopqrst'), 2) self.gen.sample(tuple('abcdefghijklmnopqrst'), 2) + population = range(sys.maxsize * 2) + sample = self.gen.sample(population, 10) + self.assertEqual(len(sample), 10) + self.assertEqual(len(set(sample)), 10) + self.assertTrue(all(element in population for element in sample)) def test_sample_on_dicts(self): self.assertRaises(TypeError, self.gen.sample, dict.fromkeys('abcdef'), 2) diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 2c9c290e8906b73..a1840112963c02d 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -162,24 +162,14 @@ def test_large_operands(self): def test_large_range(self): # Check long ranges (len > sys.maxsize) - # len() is expected to fail due to limitations of the __len__ protocol - def _range_len(x): - try: - length = len(x) - except OverflowError: - step = x[1] - x[0] - length = 1 + ((x[-1] - x[0]) // step) - return length - a = -sys.maxsize b = sys.maxsize expected_len = b - a x = range(a, b) self.assertIn(a, x) self.assertNotIn(b, x) - self.assertRaises(OverflowError, len, x) + self.assertEqual(len(x), expected_len) self.assertTrue(x) - self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 self.assertEqual(x[idx], a+idx) @@ -195,9 +185,8 @@ def _range_len(x): x = range(a, b) self.assertIn(a, x) self.assertNotIn(b, x) - self.assertRaises(OverflowError, len, x) + self.assertEqual(len(x), expected_len) self.assertTrue(x) - self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 self.assertEqual(x[idx], a+idx) @@ -214,9 +203,8 @@ def _range_len(x): x = range(a, b, c) self.assertIn(a, x) self.assertNotIn(b, x) - self.assertRaises(OverflowError, len, x) + self.assertEqual(len(x), expected_len) self.assertTrue(x) - self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 self.assertEqual(x[idx], a+(idx*c)) @@ -233,9 +221,8 @@ def _range_len(x): x = range(a, b, c) self.assertIn(a, x) self.assertNotIn(b, x) - self.assertRaises(OverflowError, len, x) + self.assertEqual(len(x), expected_len) self.assertTrue(x) - self.assertEqual(_range_len(x), expected_len) self.assertEqual(x[0], a) idx = sys.maxsize+1 self.assertEqual(x[idx], a+(idx*c)) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-29-12-25-00.gh-issue-94937.len-overflow-fallback.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-29-12-25-00.gh-issue-94937.len-overflow-fallback.rst new file mode 100644 index 000000000000000..e8dea053f6afb44 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-29-12-25-00.gh-issue-94937.len-overflow-fallback.rst @@ -0,0 +1,3 @@ +Allow :func:`len` to return values larger than :data:`sys.maxsize` when +``__len__`` can provide a Python integer fallback, including for large +:class:`range` objects. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 3bdc16437e2bc61..2b4ebee84e01777 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -3880,13 +3880,8 @@ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - Py_ssize_t len_i = PyObject_Length(arg_o); + PyObject *res_o = _PyObject_LengthAsPyLong(arg_o); _PyFrame_StackPointerInvalidate(frame); - if (len_i < 0) { - JUMP_TO_LABEL(error); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6962,12 +6957,8 @@ obj = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + PyObject *len_o = _PyObject_LengthAsPyLong(PyStackRef_AsPyObjectBorrow(obj)); _PyFrame_StackPointerInvalidate(frame); - if (len_i < 0) { - JUMP_TO_LABEL(error); - } - PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) { JUMP_TO_LABEL(error); } diff --git a/Objects/abstract.c b/Objects/abstract.c index da2a0b0c285b52b..f44f957c573df64 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -80,6 +80,61 @@ PyObject_Length(PyObject *o) } #define PyObject_Length PyObject_Size +static PyObject * +object_length_from_special(PyObject *o) +{ + PyObject *meth = _PyObject_LookupSpecial(o, &_Py_ID(__len__)); + if (meth == NULL) { + if (!PyErr_Occurred()) { + type_error("object of type '%.200s' has no len()", o); + } + return NULL; + } + + PyObject *len = _PyObject_CallNoArgs(meth); + Py_DECREF(meth); + if (len == NULL) { + return NULL; + } + + Py_SETREF(len, PyNumber_Index(len)); + if (len == NULL) { + return NULL; + } + + assert(PyLong_Check(len)); + if (_PyLong_IsNegative((PyLongObject *)len)) { + Py_DECREF(len); + PyErr_SetString(PyExc_ValueError, + "__len__() should return >= 0"); + return NULL; + } + return len; +} + +PyObject * +_PyObject_LengthAsPyLong(PyObject *o) +{ + /* Heap types use the generic slot wrapper, which would discard a large + Python integer after calling __len__. Call the special method directly + so it is evaluated exactly once. */ + if (PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HEAPTYPE)) { + return object_length_from_special(o); + } + + Py_ssize_t res = PyObject_Size(o); + if (res >= 0) { + return PyLong_FromSsize_t(res); + } + assert(PyErr_Occurred()); + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) { + return NULL; + } + + PyErr_Clear(); + return object_length_from_special(o); +} + int _PyObject_HasLen(PyObject *o) { return (Py_TYPE(o)->tp_as_sequence && Py_TYPE(o)->tp_as_sequence->sq_length) || diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 55b7f108730728d..edbb09537db298a 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -329,6 +329,13 @@ range_length(PyObject *op) return PyLong_AsSsize_t(r->length); } +static PyObject * +range_len(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + rangeobject *r = (rangeobject*)op; + return Py_NewRef(r->length); +} + static PyObject * compute_item(rangeobject *r, PyObject *i) { @@ -779,6 +786,7 @@ PyDoc_STRVAR(index_doc, "Raise ValueError if the value is not present."); static PyMethodDef range_methods[] = { + {"__len__", range_len, METH_NOARGS | METH_COEXIST, NULL}, {"__reversed__", range_reverse, METH_NOARGS, reverse_doc}, {"__reduce__", range_reduce, METH_NOARGS}, {"count", range_count, METH_O, count_doc}, diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 796897ff24cdc02..21b0aefdd117168 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1,6 +1,7 @@ /* Built-in functions */ #include "Python.h" +#include "pycore_abstract.h" // _PyObject_LengthAsPyLong() #include "pycore_ast.h" // _PyAST_Validate() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_cell.h" // PyCell_GetRef() @@ -1976,14 +1977,7 @@ static PyObject * builtin_len(PyObject *module, PyObject *obj) /*[clinic end generated code: output=fa7a270d314dfb6c input=bc55598da9e9c9b5]*/ { - Py_ssize_t res; - - res = PyObject_Size(obj); - if (res < 0) { - assert(PyErr_Occurred()); - return NULL; - } - return PyLong_FromSsize_t(res); + return _PyObject_LengthAsPyLong(obj); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 31eaeab0d67841f..66b1a69dcace46d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -7,7 +7,7 @@ // See Tools/cases_generator/README.md for more information. #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_abstract.h" // _PyIndex_Check(), _PyObject_LengthAsPyLong() #include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" #include "pycore_call.h" // _Py_METH_CALL_FLAGS @@ -3636,9 +3636,7 @@ dummy_func( inst(GET_LEN, (obj -- obj, len)) { // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - ERROR_IF(len_i < 0); - PyObject *len_o = PyLong_FromSsize_t(len_i); + PyObject *len_o = _PyObject_LengthAsPyLong(PyStackRef_AsPyObjectBorrow(obj)); ERROR_IF(len_o == NULL); len = PyStackRef_FromPyObjectSteal(len_o); } @@ -4967,12 +4965,7 @@ dummy_func( /* len(o) */ STAT_INC(CALL, hit); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - Py_ssize_t len_i = PyObject_Length(arg_o); - if (len_i < 0) { - ERROR_NO_POP(); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + PyObject *res_o = _PyObject_LengthAsPyLong(arg_o); if (res_o == NULL) { ERROR_NO_POP(); } diff --git a/Python/ceval.h b/Python/ceval.h index 00db67265a4f3a1..a02c73c1037aa0e 100644 --- a/Python/ceval.h +++ b/Python/ceval.h @@ -1,7 +1,7 @@ #define _PY_INTERPRETER #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_abstract.h" // _PyIndex_Check(), _PyObject_LengthAsPyLong() #include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" #include "pycore_call.h" // _PyObject_CallNoArgs() diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c5b2dfcf5f618f3..29eeeb8b1e75eb9 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -14370,13 +14370,8 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + PyObject *len_o = _PyObject_LengthAsPyLong(PyStackRef_AsPyObjectBorrow(obj)); _PyFrame_StackPointerInvalidate(frame); - if (len_i < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18778,14 +18773,8 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - Py_ssize_t len_i = PyObject_Length(arg_o); + PyObject *res_o = _PyObject_LengthAsPyLong(arg_o); _PyFrame_StackPointerInvalidate(frame); - if (len_i < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dd0ce41e4b06b4a..26785c0acaad208 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3880,13 +3880,8 @@ PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - Py_ssize_t len_i = PyObject_Length(arg_o); + PyObject *res_o = _PyObject_LengthAsPyLong(arg_o); _PyFrame_StackPointerInvalidate(frame); - if (len_i < 0) { - JUMP_TO_LABEL(error); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6962,12 +6957,8 @@ obj = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + PyObject *len_o = _PyObject_LengthAsPyLong(PyStackRef_AsPyObjectBorrow(obj)); _PyFrame_StackPointerInvalidate(frame); - if (len_i < 0) { - JUMP_TO_LABEL(error); - } - PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) { JUMP_TO_LABEL(error); } diff --git a/Tools/jit/template.c b/Tools/jit/template.c index f1e2e25aaecdada..5c80c60b491f28c 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -1,6 +1,8 @@ #include "Python.h" +#include "pycore_abstract.h" + #ifndef NDEBUG #undef assert #define assert(TEST) ((TEST) ? 0 : _Py_jit_assertion_failure(__LINE__))