From 07130345a9fc76873bdaf34c3929b13099bad4d0 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 15 Nov 2024 12:14:28 +0100 Subject: [PATCH 01/19] Add freelist of compact int objects --- Include/internal/pycore_freelist_state.h | 2 ++ Include/internal/pycore_long.h | 2 ++ Objects/longobject.c | 36 +++++++++++------------- Python/bytecodes.c | 21 +++++++------- 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 4e04cf431e0b31..a1a94c1f2dc880 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -14,6 +14,7 @@ extern "C" { # define Py_dicts_MAXFREELIST 80 # define Py_dictkeys_MAXFREELIST 80 # define Py_floats_MAXFREELIST 100 +# define Py_ints_MAXFREELIST 100 # define Py_slices_MAXFREELIST 1 # define Py_contexts_MAXFREELIST 255 # define Py_async_gens_MAXFREELIST 80 @@ -35,6 +36,7 @@ struct _Py_freelist { struct _Py_freelists { struct _Py_freelist floats; + struct _Py_freelist ints; struct _Py_freelist tuples[PyTuple_MAXSAVESIZE]; struct _Py_freelist lists; struct _Py_freelist dicts; diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 196b4152280a35..e17a9dbf2d9c72 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -55,6 +55,8 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); /* other API */ +void _PyLong_Free(PyLongObject *op); + #define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints) // _PyLong_GetZero() and _PyLong_GetOne() must always be available diff --git a/Objects/longobject.c b/Objects/longobject.c index 4aa35685b509f2..2fd931afa39ceb 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6,6 +6,7 @@ #include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_call.h" // _PyObject_MakeTpCall +#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS @@ -221,10 +222,14 @@ _PyLong_FromMedium(sdigit x) assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); /* We could use a freelist here */ - PyLongObject *v = PyObject_Malloc(sizeof(PyLongObject)); + + PyLongObject *v = _Py_FREELIST_POP(PyLongObject, ints); if (v == NULL) { - PyErr_NoMemory(); - return NULL; + v = PyObject_Malloc(sizeof(PyLongObject)); + if (v == NULL) { + PyErr_NoMemory(); + return NULL; + } } digit abs_x = x < 0 ? -x : x; _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); @@ -3614,24 +3619,17 @@ long_richcompare(PyObject *self, PyObject *other, int op) static void long_dealloc(PyObject *self) { - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref small Ints out of existence. Instead, - * since small Ints are immortal, re-set the reference count. - */ - PyLongObject *pylong = (PyLongObject*)self; - if (pylong && _PyLong_IsCompact(pylong)) { - stwodigits ival = medium_value(pylong); - if (IS_SMALL_INT(ival)) { - PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); - if (pylong == small_pylong) { - _Py_SetImmortal(self); - return; - } - } - } Py_TYPE(self)->tp_free(self); } +void _PyLong_Free(PyLongObject *op) { + if (_PyLong_IsCompact(op)) { + _Py_FREELIST_FREE(ints, op, PyObject_Free); + return; + } + PyObject_Free(op); +} + static Py_hash_t long_hash(PyObject *obj) { @@ -6615,7 +6613,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_init */ 0, /* tp_alloc */ long_new, /* tp_new */ - PyObject_Free, /* tp_free */ + _PyLong_Free, /* tp_free */ .tp_vectorcall = long_vectorcall, .tp_version_tag = _Py_TYPE_VERSION_INT, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 04983fd861ec59..141ba5a6868d09 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -26,6 +26,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject +#include "pycore_long.h" // void _PyLong_Free(PyLongObject *op); #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_tuple.h" // _PyTuple_ITEMS() @@ -514,8 +515,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -527,8 +528,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -540,8 +541,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -797,7 +798,7 @@ dummy_func( PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); DEAD(sub_st); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -817,7 +818,7 @@ dummy_func( DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); DEAD(sub_st); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -838,7 +839,7 @@ dummy_func( PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); DEAD(sub_st); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -950,7 +951,7 @@ dummy_func( PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); DEAD(sub_st); PyStackRef_CLOSE(list_st); } From be58ade742e286cc1e957257c11eaf2de4f2f6e9 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 15 Nov 2024 12:32:12 +0100 Subject: [PATCH 02/19] fix build --- Objects/longobject.c | 4 ++-- Python/executor_cases.c.h | 20 ++++++++++---------- Python/generated_cases.c.h | 20 ++++++++++---------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 2fd931afa39ceb..009237c92d5e36 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -43,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)_PyLong_Free); } static inline int @@ -6613,7 +6613,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_init */ 0, /* tp_alloc */ long_new, /* tp_new */ - _PyLong_Free, /* tp_free */ + (freefunc)_PyLong_Free, /* tp_free */ .tp_vectorcall = long_vectorcall, .tp_version_tag = _Py_TYPE_VERSION_INT, }; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 494ace1bd85822..029782a233c0c9 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -626,8 +626,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -646,8 +646,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -666,8 +666,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -991,7 +991,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1033,7 +1033,7 @@ } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1072,7 +1072,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1247,7 +1247,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 77bf6ad3781f17..2e3e3a1e9c1d55 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -118,8 +118,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -287,8 +287,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -358,8 +358,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -584,7 +584,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -616,7 +616,7 @@ DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -648,7 +648,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -7707,7 +7707,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); From 3f50b54741aef4d404f8143973d68ffffc03cabb Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 15 Nov 2024 13:21:39 +0100 Subject: [PATCH 03/19] cleanup freelist at exit --- Objects/object.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/object.c b/Objects/object.c index 052dea9ad1feff..75e57391199592 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -889,6 +889,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() clear_freelist(&freelists->floats, is_finalization, free_object); + clear_freelist(&freelists->ints, is_finalization, free_object); for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) { clear_freelist(&freelists->tuples[i], is_finalization, free_object); } From fa97302acab37799217c7dbbb62792a8dbbf8457 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 16 Nov 2024 19:59:35 +0100 Subject: [PATCH 04/19] fix memory leak; align with float implementation --- Include/internal/pycore_long.h | 3 ++- Objects/longobject.c | 34 ++++++++++++++++++++++++++++++++-- Objects/object.c | 2 +- Python/bytecodes.c | 22 +++++++++++----------- Python/executor_cases.c.h | 20 ++++++++++---------- Python/generated_cases.c.h | 20 ++++++++++---------- 6 files changed, 66 insertions(+), 35 deletions(-) diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index e17a9dbf2d9c72..1d1a3c9467a090 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -55,7 +55,8 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); /* other API */ -void _PyLong_Free(PyLongObject *op); +void +_PyLong_ExactDealloc(PyObject *self); #define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints) diff --git a/Objects/longobject.c b/Objects/longobject.c index 009237c92d5e36..3d7b28a7335b22 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -43,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)_PyLong_Free); + _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)_PyLong_ExactDealloc); } static inline int @@ -3616,9 +3616,39 @@ long_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_RICHCOMPARE(result, 0, op); } +void +_PyLong_ExactDealloc(PyObject *self) +{ + assert(PyLong_CheckExact(self)); + + if (_PyLong_IsCompact((PyLongObject *)self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; + } + PyObject_Free(self); +} + static void long_dealloc(PyObject *self) { + PyLongObject *pylong = (PyLongObject*)self; + + if (pylong && _PyLong_IsCompact(pylong)) { + stwodigits ival = medium_value(pylong); + if (IS_SMALL_INT(ival)) { + PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); + if (pylong == small_pylong) { + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref small Ints out of existence. Instead, + * since small Ints are immortal, re-set the reference count. + */ + // can we remove the next two lines? the immortal objects now have a fixed refcount + // in particular in the free-threading build this seeems safe + _Py_SetImmortal(self); + return; + } + } + } Py_TYPE(self)->tp_free(self); } @@ -6613,7 +6643,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_init */ 0, /* tp_alloc */ long_new, /* tp_new */ - (freefunc)_PyLong_Free, /* tp_free */ + (freefunc)PyObject_Free, /* tp_free */ .tp_vectorcall = long_vectorcall, .tp_version_tag = _Py_TYPE_VERSION_INT, }; diff --git a/Objects/object.c b/Objects/object.c index 75e57391199592..64b7de85a6147e 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -889,7 +889,6 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() clear_freelist(&freelists->floats, is_finalization, free_object); - clear_freelist(&freelists->ints, is_finalization, free_object); for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) { clear_freelist(&freelists->tuples[i], is_finalization, free_object); } @@ -907,6 +906,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) clear_freelist(&freelists->object_stack_chunks, 1, PyMem_RawFree); } clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free); + clear_freelist(&freelists->ints, is_finalization, free_object); } /* diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 141ba5a6868d09..c69706edd84203 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -26,7 +26,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject -#include "pycore_long.h" // void _PyLong_Free(PyLongObject *op); +#include "pycore_long.h" // void _PyLong_ExactDealloc(PyLongObject *op); #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_tuple.h" // _PyTuple_ITEMS() @@ -515,8 +515,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -528,8 +528,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -541,8 +541,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -798,7 +798,7 @@ dummy_func( PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -818,7 +818,7 @@ dummy_func( DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -839,7 +839,7 @@ dummy_func( PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -951,7 +951,7 @@ dummy_func( PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); DEAD(sub_st); PyStackRef_CLOSE(list_st); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 029782a233c0c9..f7055eba714075 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -626,8 +626,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -646,8 +646,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -666,8 +666,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -991,7 +991,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1033,7 +1033,7 @@ } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1072,7 +1072,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1247,7 +1247,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2e3e3a1e9c1d55..73f376e4a870da 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -118,8 +118,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -287,8 +287,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -358,8 +358,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_Free); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -584,7 +584,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -616,7 +616,7 @@ DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -648,7 +648,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -7707,7 +7707,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_Free ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); From d72486f5eeb7ca8e6f8c9523bcb4b1b0828d74ba Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 16 Nov 2024 20:00:16 +0100 Subject: [PATCH 05/19] remove stale comment --- Objects/longobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 3d7b28a7335b22..eec395351b6a4d 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -221,7 +221,6 @@ _PyLong_FromMedium(sdigit x) { assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - /* We could use a freelist here */ PyLongObject *v = _Py_FREELIST_POP(PyLongObject, ints); if (v == NULL) { From e07c2188052873c0e4298128bdbc7e92e39bb026 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 16 Nov 2024 20:01:21 +0100 Subject: [PATCH 06/19] remove unused function --- Objects/longobject.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index eec395351b6a4d..7b55fb272d16a8 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3651,14 +3651,6 @@ long_dealloc(PyObject *self) Py_TYPE(self)->tp_free(self); } -void _PyLong_Free(PyLongObject *op) { - if (_PyLong_IsCompact(op)) { - _Py_FREELIST_FREE(ints, op, PyObject_Free); - return; - } - PyObject_Free(op); -} - static Py_hash_t long_hash(PyObject *obj) { From 328e0c164346999aa43401813ce6ed55edd72a8d Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 16 Nov 2024 22:22:55 +0100 Subject: [PATCH 07/19] avoid some problematic decrefs --- Objects/longobject.c | 2 +- Python/bytecodes.c | 8 ++++---- Python/executor_cases.c.h | 8 ++++---- Python/generated_cases.c.h | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 7b55fb272d16a8..dc19d488530de0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -43,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)_PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free); // needs to be converted to freelist? } static inline int diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c69706edd84203..933d79c163c713 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -516,7 +516,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -529,7 +529,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -542,7 +542,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -951,7 +951,7 @@ dummy_func( PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f7055eba714075..d2c50b36a0edb1 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -627,7 +627,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -647,7 +647,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -667,7 +667,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1247,7 +1247,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 73f376e4a870da..b5b07d9c66a21c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -119,7 +119,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -288,7 +288,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -359,7 +359,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -7707,7 +7707,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc ); + PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); From e1dc2b3d872e91c7ac20cddf5d9aa4a617d0c255 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 16 Nov 2024 23:36:32 +0100 Subject: [PATCH 08/19] jit build --- Include/internal/pycore_long.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 1d1a3c9467a090..8bead00e70640c 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -55,8 +55,7 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); /* other API */ -void -_PyLong_ExactDealloc(PyObject *self); +PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self); #define _PyLong_SMALL_INTS _Py_SINGLETON(small_ints) From 9df776b599995e4cb8d2dab60f0382f16895ebfa Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 16 Nov 2024 22:37:49 +0000 Subject: [PATCH 09/19] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst new file mode 100644 index 00000000000000..abba62811e3758 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst @@ -0,0 +1 @@ +Increase performance of int by adding a freelist for compact ints. From db8247e0aa20aa33a17af6b342cc138117302ab4 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 21 Nov 2024 03:13:37 +0800 Subject: [PATCH 10/19] Apply review suggestions --- Objects/longobject.c | 18 ++++++++++-------- Python/bytecodes.c | 6 +++--- Python/executor_cases.c.h | 6 +++--- Python/generated_cases.c.h | 6 +++--- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index dc19d488530de0..3d16a5d1e53d9a 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -43,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor)PyObject_Free); // needs to be converted to freelist? + _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor) _PyLong_ExactDealloc); } static inline int @@ -222,7 +222,9 @@ _PyLong_FromMedium(sdigit x) assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - PyLongObject *v = _Py_FREELIST_POP(PyLongObject, ints); + // The small int cache is incompatible with _Py_NewReference which is called + // by _Py_FREELIST_POP. + PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP_MEM(ints); if (v == NULL) { v = PyObject_Malloc(sizeof(PyLongObject)); if (v == NULL) { @@ -3618,13 +3620,13 @@ long_richcompare(PyObject *self, PyObject *other, int op) void _PyLong_ExactDealloc(PyObject *self) { - assert(PyLong_CheckExact(self)); - - if (_PyLong_IsCompact((PyLongObject *)self)) { - _Py_FREELIST_FREE(ints, self, PyObject_Free); - return; + if (PyLong_CheckExact(self)) { + if (_PyLong_IsCompact((PyLongObject *)self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; + } } - PyObject_Free(self); + Py_TYPE(self)->tp_free(self); } static void diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f24ca0f62755d9..7d61fc48e21ea4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -516,7 +516,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -529,7 +529,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -542,7 +542,7 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2a0e07fb1e8c2b..73be53d5a2ce46 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -627,7 +627,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -647,7 +647,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -667,7 +667,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e25d8b4785125c..1fb36cdde1a1c2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -119,7 +119,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -288,7 +288,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -359,7 +359,7 @@ STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); // needs to be converted to freelist + PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } From d1e4aa268ed83dfcb35b18b040451a75bd003bee Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 21 Nov 2024 03:20:15 +0800 Subject: [PATCH 11/19] Fixup --- Objects/longobject.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 3d16a5d1e53d9a..d3cdba4365d4c2 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3620,13 +3620,11 @@ long_richcompare(PyObject *self, PyObject *other, int op) void _PyLong_ExactDealloc(PyObject *self) { - if (PyLong_CheckExact(self)) { - if (_PyLong_IsCompact((PyLongObject *)self)) { - _Py_FREELIST_FREE(ints, self, PyObject_Free); - return; - } + if (_PyLong_IsCompact((PyLongObject *)self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; } - Py_TYPE(self)->tp_free(self); + PyObject_Free(self); } static void @@ -3650,6 +3648,14 @@ long_dealloc(PyObject *self) } } } + + if (PyLong_CheckExact(self)) { + if (_PyLong_IsCompact((PyLongObject *)self)) { + _Py_FREELIST_FREE(ints, self, PyObject_Free); + return; + } + } + Py_TYPE(self)->tp_free(self); } From 644e85a5ae7a31b2cae09b9528928ff75d995cbf Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 21 Nov 2024 10:20:03 +0100 Subject: [PATCH 12/19] review comments part 1 --- Objects/longobject.c | 24 +++++++++++------------- Python/bytecodes.c | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index d3cdba4365d4c2..1674a2e378fd20 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6,7 +6,7 @@ #include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_call.h" // _PyObject_MakeTpCall -#include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() +#include "pycore_freelist.h" // _Py_FREELIST_FREE, _Py_FREELIST_POP #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS @@ -43,7 +43,7 @@ static inline void _Py_DECREF_INT(PyLongObject *op) { assert(PyLong_CheckExact(op)); - _Py_DECREF_SPECIALIZED((PyObject *)op, (destructor) _PyLong_ExactDealloc); + _Py_DECREF_SPECIALIZED((PyObject *)op, _PyLong_ExactDealloc); } static inline int @@ -3631,26 +3631,24 @@ static void long_dealloc(PyObject *self) { PyLongObject *pylong = (PyLongObject*)self; + assert(pylong); - if (pylong && _PyLong_IsCompact(pylong)) { + if (_PyLong_IsCompact(pylong)) { stwodigits ival = medium_value(pylong); if (IS_SMALL_INT(ival)) { PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); if (pylong == small_pylong) { /* This should never get called, but we also don't want to SEGV if - * we accidentally decref small Ints out of existence. Instead, - * since small Ints are immortal, re-set the reference count. - */ - // can we remove the next two lines? the immortal objects now have a fixed refcount - // in particular in the free-threading build this seeems safe + * we accidentally decref small Ints out of existence. Instead, + * since small Ints are immortal, re-set the reference count. + * + * With a fixed refcount for immortal objects this should not happen + */ _Py_SetImmortal(self); return; } } - } - - if (PyLong_CheckExact(self)) { - if (_PyLong_IsCompact((PyLongObject *)self)) { + if (PyLong_CheckExact(self)) { _Py_FREELIST_FREE(ints, self, PyObject_Free); return; } @@ -6642,7 +6640,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_init */ 0, /* tp_alloc */ long_new, /* tp_new */ - (freefunc)PyObject_Free, /* tp_free */ + PyObject_Free, /* tp_free */ .tp_vectorcall = long_vectorcall, .tp_version_tag = _Py_TYPE_VERSION_INT, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7d61fc48e21ea4..1112ed55000864 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -26,7 +26,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject -#include "pycore_long.h" // void _PyLong_ExactDealloc(PyLongObject *op); +#include "pycore_long.h" // _PyLong_ExactDealloc() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_tuple.h" // _PyTuple_ITEMS() From 9f86b6ef5beadeb3341f7afe856a4e5917f62f4d Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 21 Nov 2024 12:25:44 +0100 Subject: [PATCH 13/19] review comments part 2 --- Objects/longobject.c | 4 ++-- Python/bytecodes.c | 20 ++++++++++---------- Python/executor_cases.c.h | 20 ++++++++++---------- Python/generated_cases.c.h | 20 ++++++++++---------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 1674a2e378fd20..6eaa31bdcc18b3 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -222,8 +222,8 @@ _PyLong_FromMedium(sdigit x) assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - // The small int cache is incompatible with _Py_NewReference which is called - // by _Py_FREELIST_POP. + // We use _Py_FREELIST_POP_MEM instead of _Py_FREELIST_POP because the new + // reference is created in _PyObject_Init PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP_MEM(ints); if (v == NULL) { v = PyObject_Malloc(sizeof(PyLongObject)); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1112ed55000864..7b5ccc0708b17a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -515,8 +515,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -528,8 +528,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -541,8 +541,8 @@ dummy_func( STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); INPUTS_DEAD(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); @@ -798,7 +798,7 @@ dummy_func( PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -818,7 +818,7 @@ dummy_func( DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -839,7 +839,7 @@ dummy_func( PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -951,7 +951,7 @@ dummy_func( PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); DEAD(sub_st); PyStackRef_CLOSE(list_st); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 73be53d5a2ce46..0e5cdbfd806d51 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -626,8 +626,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -646,8 +646,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -666,8 +666,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) JUMP_TO_ERROR(); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -991,7 +991,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1033,7 +1033,7 @@ } STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1072,7 +1072,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -1247,7 +1247,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1fb36cdde1a1c2..95bc25e022dd2d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -118,8 +118,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -287,8 +287,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -358,8 +358,8 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(BINARY_OP, hit); PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)_PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); if (res_o == NULL) goto pop_2_error; res = PyStackRef_FromPyObjectSteal(res_o); } @@ -584,7 +584,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -616,7 +616,7 @@ DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(str_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -648,7 +648,7 @@ PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); Py_INCREF(res_o); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(tuple_st); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; @@ -7707,7 +7707,7 @@ PyList_SET_ITEM(list, index, PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); Py_DECREF(old_value); - PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)_PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); PyStackRef_CLOSE(list_st); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); From 1e548bdced82f085b9baedf867e25180ba385d0a Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 23 Nov 2024 21:48:41 +0100 Subject: [PATCH 14/19] review suggestion --- Objects/longobject.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 6eaa31bdcc18b3..5b3f8167871d24 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -222,19 +222,17 @@ _PyLong_FromMedium(sdigit x) assert(!IS_SMALL_INT(x)); assert(is_medium_int(x)); - // We use _Py_FREELIST_POP_MEM instead of _Py_FREELIST_POP because the new - // reference is created in _PyObject_Init - PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP_MEM(ints); + PyLongObject *v = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); if (v == NULL) { v = PyObject_Malloc(sizeof(PyLongObject)); if (v == NULL) { PyErr_NoMemory(); return NULL; } + _PyObject_Init((PyObject*)v, &PyLong_Type); } digit abs_x = x < 0 ? -x : x; _PyLong_SetSignAndDigitCount(v, x<0?-1:1, 1); - _PyObject_Init((PyObject*)v, &PyLong_Type); v->long_value.ob_digit[0] = abs_x; return (PyObject*)v; } From 60220c05e6433f61ae8f6495f3f50e95f263aaec Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 2 Dec 2024 16:51:28 +0100 Subject: [PATCH 15/19] add small int check to _PyLong_ExactDealloc --- Objects/longobject.c | 50 +++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 5b3f8167871d24..fd4c2d515ae38b 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3615,10 +3615,32 @@ long_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_RICHCOMPARE(result, 0, op); } +static inline int +_PyLong_IsSmallInt(PyObject *self) +{ + PyLongObject *pylong = (PyLongObject *)self; + assert(_PyLong_IsCompact(pylong)); + stwodigits ival = medium_value(pylong); + if (IS_SMALL_INT(ival)) { + PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); + if (pylong == small_pylong) { + return 1; + } + } + return 0; +} + void _PyLong_ExactDealloc(PyObject *self) { if (_PyLong_IsCompact((PyLongObject *)self)) { + #ifndef Py_GIL_DISABLED + if (_PyLong_IsSmallInt(self)) { + // See PEP 683, section Accidental De-Immortalizing for details + _Py_SetImmortal(self); + return; + } + #endif _Py_FREELIST_FREE(ints, self, PyObject_Free); return; } @@ -3628,23 +3650,17 @@ _PyLong_ExactDealloc(PyObject *self) static void long_dealloc(PyObject *self) { - PyLongObject *pylong = (PyLongObject*)self; - assert(pylong); - - if (_PyLong_IsCompact(pylong)) { - stwodigits ival = medium_value(pylong); - if (IS_SMALL_INT(ival)) { - PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); - if (pylong == small_pylong) { - /* This should never get called, but we also don't want to SEGV if - * we accidentally decref small Ints out of existence. Instead, - * since small Ints are immortal, re-set the reference count. - * - * With a fixed refcount for immortal objects this should not happen - */ - _Py_SetImmortal(self); - return; - } + assert(self); + if (_PyLong_IsCompact((PyLongObject *)self)) { + if (_PyLong_IsSmallInt(self)) { + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref small Ints out of existence. Instead, + * since small Ints are immortal, re-set the reference count. + * + * See PEP 683, section Accidental De-Immortalizing for details + */ + _Py_SetImmortal(self); + return; } if (PyLong_CheckExact(self)) { _Py_FREELIST_FREE(ints, self, PyObject_Free); From 593d62182619db75659f4134bfb533db6c656438 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 2 Dec 2024 17:03:06 +0100 Subject: [PATCH 16/19] add COMPARE_OP_INT --- Objects/longobject.c | 1 + Python/bytecodes.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index fd4c2d515ae38b..0061b12aa22d88 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3633,6 +3633,7 @@ _PyLong_IsSmallInt(PyObject *self) void _PyLong_ExactDealloc(PyObject *self) { + assert(PyLong_CheckExact(self)); if (_PyLong_IsCompact((PyLongObject *)self)) { #ifndef Py_GIL_DISABLED if (_PyLong_IsSmallInt(self)) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 67af0096527d01..dbcb3078ddb45d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2477,9 +2477,9 @@ dummy_func( Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); DEAD(left); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); DEAD(right); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. From efde111a1b5ada77534807d5ce80bc9d668a7488 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 7 Dec 2024 23:11:19 +0100 Subject: [PATCH 17/19] review comment --- Objects/longobject.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 0061b12aa22d88..96d59f542a7c3c 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3616,7 +3616,7 @@ long_richcompare(PyObject *self, PyObject *other, int op) } static inline int -_PyLong_IsSmallInt(PyObject *self) +compact_int_is_small(PyObject *self) { PyLongObject *pylong = (PyLongObject *)self; assert(_PyLong_IsCompact(pylong)); @@ -3636,7 +3636,7 @@ _PyLong_ExactDealloc(PyObject *self) assert(PyLong_CheckExact(self)); if (_PyLong_IsCompact((PyLongObject *)self)) { #ifndef Py_GIL_DISABLED - if (_PyLong_IsSmallInt(self)) { + if (compact_int_is_small(self)) { // See PEP 683, section Accidental De-Immortalizing for details _Py_SetImmortal(self); return; @@ -3653,7 +3653,7 @@ long_dealloc(PyObject *self) { assert(self); if (_PyLong_IsCompact((PyLongObject *)self)) { - if (_PyLong_IsSmallInt(self)) { + if (compact_int_is_small(self)) { /* This should never get called, but we also don't want to SEGV if * we accidentally decref small Ints out of existence. Instead, * since small Ints are immortal, re-set the reference count. From 437c24c66d9429911ab9dba0c979aff2272e2790 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 7 Dec 2024 23:19:13 +0100 Subject: [PATCH 18/19] regenerate --- Python/executor_cases.c.h | 4 ++-- Python/generated_cases.c.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 90ee79aeb33a5c..bf970aa42df0cd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3075,8 +3075,8 @@ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. stack_pointer[-2] = res; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2dafb0cd7de89b..c3c58d9972346a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3337,8 +3337,8 @@ Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, (destructor)PyObject_Free); - PyStackRef_CLOSE_SPECIALIZED(right, (destructor)PyObject_Free); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. } From b0349488f2e4239864517ead91281accb2c0d920 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Sat, 7 Dec 2024 23:46:05 +0100 Subject: [PATCH 19/19] Update Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst --- .../2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst index abba62811e3758..fd1570908c1fd6 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-22-37-46.gh-issue-126868.yOoHSY.rst @@ -1 +1 @@ -Increase performance of int by adding a freelist for compact ints. +Increase performance of :class:`int` by adding a freelist for compact ints. pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy