From f99a3d976cf1e485531fa7f9820c9196806aaaf0 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 09:22:27 +0900 Subject: [PATCH 01/55] test --- Lib/test/datetimetester.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index ecb37250ceb6c4..200f8489feb020 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7269,7 +7269,28 @@ def test_update_type_cache(self): assert isinstance(_datetime.timezone.utc, _datetime.tzinfo) del sys.modules['_datetime'] """) - script_helper.assert_python_ok('-c', script) + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) + + def test_module_state_at_shutdown(self): + # gh-132413 + script = textwrap.dedent(""" + import sys + import _datetime + + def gen(): + try: + yield + finally: + assert not sys.modules + td = _datetime.timedelta(days=1) # crash + assert td.days == 1 + + it = gen() + next(it) + """) + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) def load_tests(loader, standard_tests, pattern): From f502244db3ae6c7aaabf1f822059b534e4d7a497 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 09:25:52 +0900 Subject: [PATCH 02/55] Create a module at shutdown rather than import --- Modules/_datetimemodule.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9bba0e3354b26b..1e0b881f99cffe 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -158,6 +158,7 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) } static PyModuleDef datetimemodule; +static int set_current_module(PyInterpreterState *interp, PyObject *mod); static datetime_state * _get_current_state(PyObject **p_mod) @@ -173,7 +174,28 @@ _get_current_state(PyObject **p_mod) * so we must re-import the module. */ mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { - return NULL; + PyErr_Clear(); + /* Create a module at shutdown */ + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + return NULL; + } + PyObject *spec; + if (PyDict_GetItemStringRef(dict, "datetime_module_spec", &spec) != 1) { + return NULL; + } + mod = PyModule_FromDefAndSpec(&datetimemodule, spec); + if (mod == NULL) { + Py_DECREF(spec); + return NULL; + } + Py_DECREF(spec); + + /* The module will be held by heaptypes. Prefer + /* it not to be cached in the interp-dict. */ + if (PyModule_ExecDef(mod, &datetimemodule) < 0) { + return NULL; + } } } datetime_state *st = get_module_state(mod); @@ -198,6 +220,19 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) if (ref == NULL) { return -1; } + + if (!PyDict_ContainsString(dict, "datetime_module_spec")) { + PyObject *spec; + if (PyDict_GetItemRef(PyModule_GetDict(mod), &_Py_ID(__spec__), &spec) != 1) { + return -1; + } + if (PyDict_SetItemString(dict, "datetime_module_spec", spec) < 0) { + Py_DECREF(spec); + return -1; + } + Py_DECREF(spec); + } + int rc = PyDict_SetItem(dict, INTERP_KEY, ref); Py_DECREF(ref); return rc; From 528882d1a04a47158b7a77306c58961a0aa27f35 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 09:27:54 +0900 Subject: [PATCH 03/55] NEWS --- .../next/Library/2025-04-16-14-34-04.gh-issue-132413.TvpIn2.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-04-16-14-34-04.gh-issue-132413.TvpIn2.rst diff --git a/Misc/NEWS.d/next/Library/2025-04-16-14-34-04.gh-issue-132413.TvpIn2.rst b/Misc/NEWS.d/next/Library/2025-04-16-14-34-04.gh-issue-132413.TvpIn2.rst new file mode 100644 index 00000000000000..3e778bf54f8e02 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-16-14-34-04.gh-issue-132413.TvpIn2.rst @@ -0,0 +1 @@ +Fix crash in C version of :mod:`datetime` when used during interpreter shutdown. From 72991c2c24f6b0537ab8c95a669dd5c5382f2fec Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 09:45:43 +0900 Subject: [PATCH 04/55] Fix comment --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 1e0b881f99cffe..d42b3490854bdf 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -192,7 +192,7 @@ _get_current_state(PyObject **p_mod) Py_DECREF(spec); /* The module will be held by heaptypes. Prefer - /* it not to be cached in the interp-dict. */ + * it not to be cached in the interp-dict. */ if (PyModule_ExecDef(mod, &datetimemodule) < 0) { return NULL; } From 4abbc7374f5030a5910461fe400cd5c482437b9d Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 10:13:38 +0900 Subject: [PATCH 05/55] Remove unused code --- Modules/_datetimemodule.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index d42b3490854bdf..e5dc2afe2528f2 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -158,7 +158,6 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) } static PyModuleDef datetimemodule; -static int set_current_module(PyInterpreterState *interp, PyObject *mod); static datetime_state * _get_current_state(PyObject **p_mod) @@ -192,7 +191,7 @@ _get_current_state(PyObject **p_mod) Py_DECREF(spec); /* The module will be held by heaptypes. Prefer - * it not to be cached in the interp-dict. */ + * it not to be stored in the interp-dict. */ if (PyModule_ExecDef(mod, &datetimemodule) < 0) { return NULL; } From db2cb883806b84b0116c9e6f2b38695cb39e417b Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 10:48:23 +0900 Subject: [PATCH 06/55] Remove PyDict_Contains() --- Modules/_datetimemodule.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index e5dc2afe2528f2..1b15baaf799099 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -219,18 +219,16 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) if (ref == NULL) { return -1; } - - if (!PyDict_ContainsString(dict, "datetime_module_spec")) { - PyObject *spec; - if (PyDict_GetItemRef(PyModule_GetDict(mod), &_Py_ID(__spec__), &spec) != 1) { - return -1; - } - if (PyDict_SetItemString(dict, "datetime_module_spec", spec) < 0) { - Py_DECREF(spec); - return -1; - } + /* A module spec remains in the dict */ + PyObject *spec; + if (PyDict_GetItemRef(PyModule_GetDict(mod), &_Py_ID(__spec__), &spec) != 1) { + return -1; + } + if (PyDict_SetItemString(dict, "datetime_module_spec", spec) < 0) { Py_DECREF(spec); + return -1; } + Py_DECREF(spec); int rc = PyDict_SetItem(dict, INTERP_KEY, ref); Py_DECREF(ref); From 7225ec267517f23f4225a9c6d8e0324cd94c36df Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 18:20:26 +0900 Subject: [PATCH 07/55] Add assertion in test --- Lib/test/datetimetester.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 200f8489feb020..03b80607ee6713 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7285,6 +7285,7 @@ def gen(): assert not sys.modules td = _datetime.timedelta(days=1) # crash assert td.days == 1 + assert not sys.modules it = gen() next(it) From 22c4764584ee97d2ce3e8a0343ef41ccb95165b2 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 19:04:32 +0900 Subject: [PATCH 08/55] Correct PyModule_GetDict() usage --- Modules/_datetimemodule.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 1b15baaf799099..ed31c79ce6c4c6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -191,7 +191,7 @@ _get_current_state(PyObject **p_mod) Py_DECREF(spec); /* The module will be held by heaptypes. Prefer - * it not to be stored in the interp-dict. */ + * it not to be stored in the interpreter's dict. */ if (PyModule_ExecDef(mod, &datetimemodule) < 0) { return NULL; } @@ -219,9 +219,13 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) if (ref == NULL) { return -1; } - /* A module spec remains in the dict */ + /* A module spec remains in the interpreter's dict. */ + PyObject *mod_dict = PyModule_GetDict(mod); + if (mod_dict == NULL) { + return -1; + } PyObject *spec; - if (PyDict_GetItemRef(PyModule_GetDict(mod), &_Py_ID(__spec__), &spec) != 1) { + if (PyDict_GetItemRef(mod_dict, &_Py_ID(__spec__), &spec) != 1) { return -1; } if (PyDict_SetItemString(dict, "datetime_module_spec", spec) < 0) { From b9eaee159019eccd8e38ffac6168cda89f599a6a Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 18 Apr 2025 19:46:38 +0900 Subject: [PATCH 09/55] Fix comment --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index ed31c79ce6c4c6..23bc5360de56d7 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -219,7 +219,7 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) if (ref == NULL) { return -1; } - /* A module spec remains in the interpreter's dict. */ + /* Module spec remains in the interpreter's dict. */ PyObject *mod_dict = PyModule_GetDict(mod); if (mod_dict == NULL) { return -1; From 97a6d0d7824ba4c693fdddaa7fd16f7c77a2d865 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sat, 19 Apr 2025 04:23:51 +0900 Subject: [PATCH 10/55] Set a spec later in set_current_module() --- Modules/_datetimemodule.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 23bc5360de56d7..9736ed795196da 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -219,7 +219,14 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) if (ref == NULL) { return -1; } - /* Module spec remains in the interpreter's dict. */ + if (PyDict_SetItem(dict, INTERP_KEY, ref) < 0) { + Py_DECREF(ref); + return -1; + } + Py_DECREF(ref); + + /* Make the module spec remain in the interpreter's dict. Not required, + * but reserve the new one for memory efficiency. */ PyObject *mod_dict = PyModule_GetDict(mod); if (mod_dict == NULL) { return -1; @@ -233,10 +240,7 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) return -1; } Py_DECREF(spec); - - int rc = PyDict_SetItem(dict, INTERP_KEY, ref); - Py_DECREF(ref); - return rc; + return 0; } static void From c10d4dbb31e048c252e5e7eb022ed0a10f2f871b Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 21 Apr 2025 07:51:21 +0900 Subject: [PATCH 11/55] Hold current module until interp-end --- Include/internal/pycore_interp_structs.h | 2 + Modules/_datetimemodule.c | 179 +++++++---------------- 2 files changed, 55 insertions(+), 126 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 573b56a57e1d54..a0901386434440 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -947,6 +947,8 @@ struct _is { _Py_hashtable_t *closed_stackrefs_table; # endif #endif + + void *datetime_module_state; }; diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9736ed795196da..81b5000a841e1c 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -49,6 +49,10 @@ typedef struct { /* The interned Unix epoch datetime instance */ PyObject *epoch; + + /* Reference to the interpreter's dict where the current module will be + * reserved even after the referent dict becomes NULL at shutdown. */ + PyObject *interp_dict; } datetime_state; /* The module has a fixed number of static objects, due to being exposed @@ -133,18 +137,13 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) if (dict == NULL) { goto error; } - PyObject *ref = NULL; - if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + if (PyDict_GetItemRef(dict, INTERP_KEY, &mod) < 0) { goto error; } - if (ref != NULL) { + if (mod != NULL) { reloading = 1; - if (ref != Py_None) { - (void)PyWeakref_GetRef(ref, &mod); - if (mod == Py_None) { - Py_CLEAR(mod); - } - Py_DECREF(ref); + if (mod == Py_None) { + mod = NULL; } } if (p_reloading != NULL) { @@ -159,117 +158,51 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) static PyModuleDef datetimemodule; -static datetime_state * -_get_current_state(PyObject **p_mod) +static inline datetime_state * +get_current_state() { PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *mod = get_current_module(interp, NULL); - if (mod == NULL) { - assert(!PyErr_Occurred()); - if (PyErr_Occurred()) { - return NULL; - } - /* The static types can outlive the module, - * so we must re-import the module. */ - mod = PyImport_ImportModule("_datetime"); - if (mod == NULL) { - PyErr_Clear(); - /* Create a module at shutdown */ - PyObject *dict = PyInterpreterState_GetDict(interp); - if (dict == NULL) { - return NULL; - } - PyObject *spec; - if (PyDict_GetItemStringRef(dict, "datetime_module_spec", &spec) != 1) { - return NULL; - } - mod = PyModule_FromDefAndSpec(&datetimemodule, spec); - if (mod == NULL) { - Py_DECREF(spec); - return NULL; - } - Py_DECREF(spec); - - /* The module will be held by heaptypes. Prefer - * it not to be stored in the interpreter's dict. */ - if (PyModule_ExecDef(mod, &datetimemodule) < 0) { - return NULL; - } - } - } - datetime_state *st = get_module_state(mod); - *p_mod = mod; - return st; + void *state = interp->datetime_module_state; + assert(state != NULL); + return (datetime_state *)state; } -#define GET_CURRENT_STATE(MOD_VAR) \ - _get_current_state(&MOD_VAR) -#define RELEASE_CURRENT_STATE(ST_VAR, MOD_VAR) \ - Py_DECREF(MOD_VAR) - static int -set_current_module(PyInterpreterState *interp, PyObject *mod) +set_current_module(datetime_state *st, + PyInterpreterState *interp, PyObject *mod) { assert(mod != NULL); - PyObject *dict = PyInterpreterState_GetDict(interp); + PyObject *dict = st->interp_dict; if (dict == NULL) { return -1; } - PyObject *ref = PyWeakref_NewRef(mod, NULL); - if (ref == NULL) { - return -1; - } - if (PyDict_SetItem(dict, INTERP_KEY, ref) < 0) { - Py_DECREF(ref); - return -1; - } - Py_DECREF(ref); - - /* Make the module spec remain in the interpreter's dict. Not required, - * but reserve the new one for memory efficiency. */ - PyObject *mod_dict = PyModule_GetDict(mod); - if (mod_dict == NULL) { - return -1; - } - PyObject *spec; - if (PyDict_GetItemRef(mod_dict, &_Py_ID(__spec__), &spec) != 1) { - return -1; - } - if (PyDict_SetItemString(dict, "datetime_module_spec", spec) < 0) { - Py_DECREF(spec); + if (PyDict_SetItem(dict, INTERP_KEY, mod) < 0) { return -1; } - Py_DECREF(spec); + interp->datetime_module_state = st; return 0; } static void -clear_current_module(PyInterpreterState *interp, PyObject *expected) +clear_current_module(datetime_state *st, + PyInterpreterState *interp, PyObject *expected) { - PyObject *exc = PyErr_GetRaisedException(); - - PyObject *dict = PyInterpreterState_GetDict(interp); + PyObject *dict = st->interp_dict; if (dict == NULL) { - goto error; + return; /* Already cleared */ } + PyObject *exc = PyErr_GetRaisedException(); + if (expected != NULL) { - PyObject *ref = NULL; - if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { + PyObject *current; + if (PyDict_GetItemRef(dict, INTERP_KEY, ¤t) < 0) { goto error; } - if (ref != NULL) { - PyObject *current = NULL; - int rc = PyWeakref_GetRef(ref, ¤t); - /* We only need "current" for pointer comparison. */ - Py_XDECREF(current); - Py_DECREF(ref); - if (rc < 0) { - goto error; - } - if (current != expected) { - goto finally; - } + /* We only need "current" for pointer comparison. */ + Py_XDECREF(current); + if (current != expected) { + goto finally; } } @@ -277,6 +210,7 @@ clear_current_module(PyInterpreterState *interp, PyObject *expected) if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) { goto error; } + interp->datetime_module_state = NULL; goto finally; @@ -2147,8 +2081,7 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *x3 = NULL; PyObject *result = NULL; - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) @@ -2186,7 +2119,6 @@ delta_to_microseconds(PyDateTime_Delta *self) Py_XDECREF(x1); Py_XDECREF(x2); Py_XDECREF(x3); - RELEASE_CURRENT_STATE(st, current_mod); return result; } @@ -2226,8 +2158,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { @@ -2272,7 +2203,6 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) Done: Py_XDECREF(tuple); Py_XDECREF(num); - RELEASE_CURRENT_STATE(st, current_mod); return result; BadDivmod: @@ -2811,8 +2741,7 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *self = NULL; - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); /* Argument objects. */ PyObject *day = NULL; @@ -2917,7 +2846,6 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) Py_DECREF(x); Done: - RELEASE_CURRENT_STATE(st, current_mod); return self; #undef CLEANUP @@ -3030,12 +2958,10 @@ delta_total_seconds(PyObject *op, PyObject *Py_UNUSED(dummy)) if (total_microseconds == NULL) return NULL; - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); - RELEASE_CURRENT_STATE(st, current_mod); Py_DECREF(total_microseconds); return total_seconds; } @@ -3813,12 +3739,10 @@ date_isocalendar(PyObject *self, PyObject *Py_UNUSED(dummy)) week = 0; } - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), year, week + 1, day + 1); - RELEASE_CURRENT_STATE(st, current_mod); if (v == NULL) { return NULL; } @@ -6638,11 +6562,9 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); - RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; @@ -6882,12 +6804,10 @@ datetime_timestamp(PyObject *op, PyObject *Py_UNUSED(dummy)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - PyObject *current_mod = NULL; - datetime_state *st = GET_CURRENT_STATE(current_mod); + datetime_state *st = get_current_state(); PyObject *delta; delta = datetime_subtract(op, CONST_EPOCH(st)); - RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; result = delta_total_seconds(delta, NULL); @@ -7252,8 +7172,15 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) */ static int -init_state(datetime_state *st, PyObject *module, PyObject *old_module) +init_state(datetime_state *st, + PyInterpreterState *interp, PyObject *module, PyObject *old_module) { + PyObject *dict = PyInterpreterState_GetDict(interp); + if (dict == NULL) { + return -1; + } + st->interp_dict = Py_NewRef(dict); + /* Each module gets its own heap types. */ #define ADD_TYPE(FIELD, SPEC, BASE) \ do { \ @@ -7272,6 +7199,7 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) assert(old_module != module); datetime_state *st_old = get_module_state(old_module); *st = (datetime_state){ + .interp_dict = st->interp_dict, .isocalendar_date_type = st->isocalendar_date_type, .us_per_ms = Py_NewRef(st_old->us_per_ms), .us_per_second = Py_NewRef(st_old->us_per_second), @@ -7331,9 +7259,8 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) static int traverse_state(datetime_state *st, visitproc visit, void *arg) { - /* heap types */ Py_VISIT(st->isocalendar_date_type); - + Py_VISIT(st->interp_dict); return 0; } @@ -7349,6 +7276,7 @@ clear_state(datetime_state *st) Py_CLEAR(st->us_per_week); Py_CLEAR(st->seconds_per_day); Py_CLEAR(st->epoch); + Py_CLEAR(st->interp_dict); return 0; } @@ -7416,7 +7344,7 @@ _datetime_exec(PyObject *module) } } - if (init_state(st, module, old_module) < 0) { + if (init_state(st, interp, module, old_module) < 0) { goto error; } @@ -7522,7 +7450,7 @@ _datetime_exec(PyObject *module) static_assert(DI100Y == 25 * DI4Y - 1, "DI100Y"); assert(DI100Y == days_before_year(100+1)); - if (set_current_module(interp, module) < 0) { + if (set_current_module(st, interp, module) < 0) { goto error; } @@ -7556,10 +7484,9 @@ static int module_clear(PyObject *mod) { datetime_state *st = get_module_state(mod); - clear_state(st); - PyInterpreterState *interp = PyInterpreterState_Get(); - clear_current_module(interp, mod); + clear_current_module(st, interp, mod); + clear_state(st); // The runtime takes care of the static types for us. // See _PyTypes_FiniExtTypes().. From b7d27ead4dd82889fda34b160e96d1aec919f99d Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 21 Apr 2025 08:16:35 +0900 Subject: [PATCH 12/55] Fix decl --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 81b5000a841e1c..c278887d8200af 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -159,7 +159,7 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) static PyModuleDef datetimemodule; static inline datetime_state * -get_current_state() +get_current_state(void) { PyInterpreterState *interp = PyInterpreterState_Get(); void *state = interp->datetime_module_state; From 3fba89243ec6adc319383a85f666c9f658fc76d7 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 21 Apr 2025 19:39:28 +0900 Subject: [PATCH 13/55] assert interp-dict has a key --- Modules/_datetimemodule.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index c278887d8200af..0cfdd19b9eca60 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -162,9 +162,11 @@ static inline datetime_state * get_current_state(void) { PyInterpreterState *interp = PyInterpreterState_Get(); - void *state = interp->datetime_module_state; - assert(state != NULL); - return (datetime_state *)state; + datetime_state *st = interp->datetime_module_state; + assert(st != NULL); + assert(st->interp_dict != NULL); + assert(PyDict_Contains(st->interp_dict, INTERP_KEY) == 1); + return st; } static int From d5e12201ceb267814cf9a181bb75efda80e6eefa Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 22 Apr 2025 00:09:21 +0900 Subject: [PATCH 14/55] Reword --- Modules/_datetimemodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 0cfdd19b9eca60..06ad7949ab990a 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -50,8 +50,8 @@ typedef struct { /* The interned Unix epoch datetime instance */ PyObject *epoch; - /* Reference to the interpreter's dict where the current module will be - * reserved even after the referent dict becomes NULL at shutdown. */ + /* Extra reference to the interpreter's dict that will be decref'ed last + /* at shutdown. Use the dict to reserve the current module. */ PyObject *interp_dict; } datetime_state; From c0a27eb6b107304cd5bc16ed14a61646108858eb Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 22 Apr 2025 00:18:36 +0900 Subject: [PATCH 15/55] typo --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 06ad7949ab990a..3e659c1e3b5511 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -51,7 +51,7 @@ typedef struct { PyObject *epoch; /* Extra reference to the interpreter's dict that will be decref'ed last - /* at shutdown. Use the dict to reserve the current module. */ + * at shutdown. Use the dict to reserve the current module. */ PyObject *interp_dict; } datetime_state; From bf3d238d0ad4bbfe20622ddad60d5327985c8bc6 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 22 Apr 2025 02:52:51 +0900 Subject: [PATCH 16/55] Reword again --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 3e659c1e3b5511..a2d04350b3e247 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -51,7 +51,7 @@ typedef struct { PyObject *epoch; /* Extra reference to the interpreter's dict that will be decref'ed last - * at shutdown. Use the dict to reserve the current module. */ + * at shutdown. Keep the current module in the dict. */ PyObject *interp_dict; } datetime_state; From 0f5993654e3b90e9074118d1ac194fe866cf0443 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 22 Apr 2025 09:06:53 +0900 Subject: [PATCH 17/55] ditto --- Modules/_datetimemodule.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index a2d04350b3e247..d2a01ada50e386 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -50,8 +50,9 @@ typedef struct { /* The interned Unix epoch datetime instance */ PyObject *epoch; - /* Extra reference to the interpreter's dict that will be decref'ed last - * at shutdown. Keep the current module in the dict. */ + /* Extra reference to the interpreter's dict that will be decref'ed + * last at shutdown. We keep the current module in it, but don't rely + * on PyInterpreterState_GetDict() at the module's final phase. */ PyObject *interp_dict; } datetime_state; From 78c762a3ab6a94fd2936f2ef64fc8fdfccc899b4 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 05:54:01 +0900 Subject: [PATCH 18/55] Check if interp-dict is empty --- Modules/_datetimemodule.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index d2a01ada50e386..3d6d3140bfec35 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -202,6 +202,9 @@ clear_current_module(datetime_state *st, if (PyDict_GetItemRef(dict, INTERP_KEY, ¤t) < 0) { goto error; } + if (current == NULL) { + interp->datetime_module_state = NULL; + } /* We only need "current" for pointer comparison. */ Py_XDECREF(current); if (current != expected) { From 8503986aef40e237205dbfb4964c3c2bab4c6be2 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 13:30:19 +0900 Subject: [PATCH 19/55] Correct behavior --- Modules/_datetimemodule.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 3d6d3140bfec35..37d9165d67433f 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -159,14 +159,31 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) static PyModuleDef datetimemodule; -static inline datetime_state * +static datetime_state * get_current_state(void) { PyInterpreterState *interp = PyInterpreterState_Get(); datetime_state *st = interp->datetime_module_state; - assert(st != NULL); - assert(st->interp_dict != NULL); - assert(PyDict_Contains(st->interp_dict, INTERP_KEY) == 1); + if (st != NULL) { + assert(st->interp_dict != NULL); + assert(PyDict_Contains(st->interp_dict, INTERP_KEY) == 1); + return st; + } + PyObject *mod = get_current_module(interp, NULL); + if (mod == NULL) { + assert(!PyErr_Occurred()); + if (PyErr_Occurred()) { + return NULL; + } + /* The static types can outlive the module, + * so we must re-import the module. */ + mod = PyImport_ImportModule("_datetime"); + if (mod == NULL) { + return NULL; + } + } + st = get_module_state(mod); + Py_DECREF(mod); return st; } @@ -202,9 +219,6 @@ clear_current_module(datetime_state *st, if (PyDict_GetItemRef(dict, INTERP_KEY, ¤t) < 0) { goto error; } - if (current == NULL) { - interp->datetime_module_state = NULL; - } /* We only need "current" for pointer comparison. */ Py_XDECREF(current); if (current != expected) { @@ -216,7 +230,6 @@ clear_current_module(datetime_state *st, if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) { goto error; } - interp->datetime_module_state = NULL; goto finally; @@ -224,6 +237,9 @@ clear_current_module(datetime_state *st, PyErr_FormatUnraisable("Exception ignored while clearing _datetime module"); finally: + if (!expected || st == interp->datetime_module_state) { + interp->datetime_module_state = NULL; + } PyErr_SetRaisedException(exc); } From 4b92b193226e755e3479670a473ced62244d9be6 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 13:41:05 +0900 Subject: [PATCH 20/55] Revert _datetimemodule.c to main --- Modules/_datetimemodule.c | 133 +++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 61 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 37d9165d67433f..9bba0e3354b26b 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -49,11 +49,6 @@ typedef struct { /* The interned Unix epoch datetime instance */ PyObject *epoch; - - /* Extra reference to the interpreter's dict that will be decref'ed - * last at shutdown. We keep the current module in it, but don't rely - * on PyInterpreterState_GetDict() at the module's final phase. */ - PyObject *interp_dict; } datetime_state; /* The module has a fixed number of static objects, due to being exposed @@ -138,13 +133,18 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) if (dict == NULL) { goto error; } - if (PyDict_GetItemRef(dict, INTERP_KEY, &mod) < 0) { + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { goto error; } - if (mod != NULL) { + if (ref != NULL) { reloading = 1; - if (mod == Py_None) { - mod = NULL; + if (ref != Py_None) { + (void)PyWeakref_GetRef(ref, &mod); + if (mod == Py_None) { + Py_CLEAR(mod); + } + Py_DECREF(ref); } } if (p_reloading != NULL) { @@ -160,15 +160,9 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) static PyModuleDef datetimemodule; static datetime_state * -get_current_state(void) +_get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); - datetime_state *st = interp->datetime_module_state; - if (st != NULL) { - assert(st->interp_dict != NULL); - assert(PyDict_Contains(st->interp_dict, INTERP_KEY) == 1); - return st; - } PyObject *mod = get_current_module(interp, NULL); if (mod == NULL) { assert(!PyErr_Occurred()); @@ -182,47 +176,60 @@ get_current_state(void) return NULL; } } - st = get_module_state(mod); - Py_DECREF(mod); + datetime_state *st = get_module_state(mod); + *p_mod = mod; return st; } +#define GET_CURRENT_STATE(MOD_VAR) \ + _get_current_state(&MOD_VAR) +#define RELEASE_CURRENT_STATE(ST_VAR, MOD_VAR) \ + Py_DECREF(MOD_VAR) + static int -set_current_module(datetime_state *st, - PyInterpreterState *interp, PyObject *mod) +set_current_module(PyInterpreterState *interp, PyObject *mod) { assert(mod != NULL); - PyObject *dict = st->interp_dict; + PyObject *dict = PyInterpreterState_GetDict(interp); if (dict == NULL) { return -1; } - if (PyDict_SetItem(dict, INTERP_KEY, mod) < 0) { + PyObject *ref = PyWeakref_NewRef(mod, NULL); + if (ref == NULL) { return -1; } - interp->datetime_module_state = st; - return 0; + int rc = PyDict_SetItem(dict, INTERP_KEY, ref); + Py_DECREF(ref); + return rc; } static void -clear_current_module(datetime_state *st, - PyInterpreterState *interp, PyObject *expected) +clear_current_module(PyInterpreterState *interp, PyObject *expected) { - PyObject *dict = st->interp_dict; + PyObject *exc = PyErr_GetRaisedException(); + + PyObject *dict = PyInterpreterState_GetDict(interp); if (dict == NULL) { - return; /* Already cleared */ + goto error; } - PyObject *exc = PyErr_GetRaisedException(); - if (expected != NULL) { - PyObject *current; - if (PyDict_GetItemRef(dict, INTERP_KEY, ¤t) < 0) { + PyObject *ref = NULL; + if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { goto error; } - /* We only need "current" for pointer comparison. */ - Py_XDECREF(current); - if (current != expected) { - goto finally; + if (ref != NULL) { + PyObject *current = NULL; + int rc = PyWeakref_GetRef(ref, ¤t); + /* We only need "current" for pointer comparison. */ + Py_XDECREF(current); + Py_DECREF(ref); + if (rc < 0) { + goto error; + } + if (current != expected) { + goto finally; + } } } @@ -237,9 +244,6 @@ clear_current_module(datetime_state *st, PyErr_FormatUnraisable("Exception ignored while clearing _datetime module"); finally: - if (!expected || st == interp->datetime_module_state) { - interp->datetime_module_state = NULL; - } PyErr_SetRaisedException(exc); } @@ -2103,7 +2107,8 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *x3 = NULL; PyObject *result = NULL; - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) @@ -2141,6 +2146,7 @@ delta_to_microseconds(PyDateTime_Delta *self) Py_XDECREF(x1); Py_XDECREF(x2); Py_XDECREF(x3); + RELEASE_CURRENT_STATE(st, current_mod); return result; } @@ -2180,7 +2186,8 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { @@ -2225,6 +2232,7 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) Done: Py_XDECREF(tuple); Py_XDECREF(num); + RELEASE_CURRENT_STATE(st, current_mod); return result; BadDivmod: @@ -2763,7 +2771,8 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *self = NULL; - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); /* Argument objects. */ PyObject *day = NULL; @@ -2868,6 +2877,7 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) Py_DECREF(x); Done: + RELEASE_CURRENT_STATE(st, current_mod); return self; #undef CLEANUP @@ -2980,10 +2990,12 @@ delta_total_seconds(PyObject *op, PyObject *Py_UNUSED(dummy)) if (total_microseconds == NULL) return NULL; - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); + RELEASE_CURRENT_STATE(st, current_mod); Py_DECREF(total_microseconds); return total_seconds; } @@ -3761,10 +3773,12 @@ date_isocalendar(PyObject *self, PyObject *Py_UNUSED(dummy)) week = 0; } - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), year, week + 1, day + 1); + RELEASE_CURRENT_STATE(st, current_mod); if (v == NULL) { return NULL; } @@ -6584,9 +6598,11 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; @@ -6826,10 +6842,12 @@ datetime_timestamp(PyObject *op, PyObject *Py_UNUSED(dummy)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - datetime_state *st = get_current_state(); + PyObject *current_mod = NULL; + datetime_state *st = GET_CURRENT_STATE(current_mod); PyObject *delta; delta = datetime_subtract(op, CONST_EPOCH(st)); + RELEASE_CURRENT_STATE(st, current_mod); if (delta == NULL) return NULL; result = delta_total_seconds(delta, NULL); @@ -7194,15 +7212,8 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) */ static int -init_state(datetime_state *st, - PyInterpreterState *interp, PyObject *module, PyObject *old_module) +init_state(datetime_state *st, PyObject *module, PyObject *old_module) { - PyObject *dict = PyInterpreterState_GetDict(interp); - if (dict == NULL) { - return -1; - } - st->interp_dict = Py_NewRef(dict); - /* Each module gets its own heap types. */ #define ADD_TYPE(FIELD, SPEC, BASE) \ do { \ @@ -7221,7 +7232,6 @@ init_state(datetime_state *st, assert(old_module != module); datetime_state *st_old = get_module_state(old_module); *st = (datetime_state){ - .interp_dict = st->interp_dict, .isocalendar_date_type = st->isocalendar_date_type, .us_per_ms = Py_NewRef(st_old->us_per_ms), .us_per_second = Py_NewRef(st_old->us_per_second), @@ -7281,8 +7291,9 @@ init_state(datetime_state *st, static int traverse_state(datetime_state *st, visitproc visit, void *arg) { + /* heap types */ Py_VISIT(st->isocalendar_date_type); - Py_VISIT(st->interp_dict); + return 0; } @@ -7298,7 +7309,6 @@ clear_state(datetime_state *st) Py_CLEAR(st->us_per_week); Py_CLEAR(st->seconds_per_day); Py_CLEAR(st->epoch); - Py_CLEAR(st->interp_dict); return 0; } @@ -7366,7 +7376,7 @@ _datetime_exec(PyObject *module) } } - if (init_state(st, interp, module, old_module) < 0) { + if (init_state(st, module, old_module) < 0) { goto error; } @@ -7472,7 +7482,7 @@ _datetime_exec(PyObject *module) static_assert(DI100Y == 25 * DI4Y - 1, "DI100Y"); assert(DI100Y == days_before_year(100+1)); - if (set_current_module(st, interp, module) < 0) { + if (set_current_module(interp, module) < 0) { goto error; } @@ -7506,10 +7516,11 @@ static int module_clear(PyObject *mod) { datetime_state *st = get_module_state(mod); - PyInterpreterState *interp = PyInterpreterState_Get(); - clear_current_module(st, interp, mod); clear_state(st); + PyInterpreterState *interp = PyInterpreterState_Get(); + clear_current_module(interp, mod); + // The runtime takes care of the static types for us. // See _PyTypes_FiniExtTypes().. From 3d293800b390ff629dd891207873062e65d5f6b6 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 13:59:45 +0900 Subject: [PATCH 21/55] Fix crash (new attempt) --- Modules/_datetimemodule.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9bba0e3354b26b..0e515325d2bbc0 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -163,6 +163,13 @@ static datetime_state * _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); + datetime_state *st = interp->datetime_module_state; + if (st != NULL) { + PyObject *mod = PyType_GetModule(st->isocalendar_date_type); + assert(mod != NULL); + *p_mod = Py_NewRef(mod); + return st; + } PyObject *mod = get_current_module(interp, NULL); if (mod == NULL) { assert(!PyErr_Occurred()); @@ -176,7 +183,7 @@ _get_current_state(PyObject **p_mod) return NULL; } } - datetime_state *st = get_module_state(mod); + st = get_module_state(mod); *p_mod = mod; return st; } @@ -200,6 +207,9 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) } int rc = PyDict_SetItem(dict, INTERP_KEY, ref); Py_DECREF(ref); + if (rc == 0) { + interp->datetime_module_state = get_module_state(mod); + } return rc; } @@ -244,6 +254,9 @@ clear_current_module(PyInterpreterState *interp, PyObject *expected) PyErr_FormatUnraisable("Exception ignored while clearing _datetime module"); finally: + if (!expected || get_module_state(expected) == interp->datetime_module_state) { + interp->datetime_module_state = NULL; + } PyErr_SetRaisedException(exc); } From 26b8f51709dafb5562fcd05038f51d5f5597739a Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 19:26:22 +0900 Subject: [PATCH 22/55] Non-NULL check before PyType_Check() --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 0e515325d2bbc0..66e75e94b342cf 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -164,7 +164,7 @@ _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); datetime_state *st = interp->datetime_module_state; - if (st != NULL) { + if (st != NULL && st->isocalendar_date_type != NULL) { PyObject *mod = PyType_GetModule(st->isocalendar_date_type); assert(mod != NULL); *p_mod = Py_NewRef(mod); From 3e2ef0d9d62bf0ac5f217081829fbe2d2dff30c9 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 23 Apr 2025 21:44:13 +0900 Subject: [PATCH 23/55] Faster _get_current_state() by 6%- --- Modules/_datetimemodule.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 66e75e94b342cf..8c62c073cc335e 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -35,6 +35,9 @@ static PyTypeObject PyDateTime_TimeZoneType; typedef struct { + /* Corresponding module */ + PyObject *module; + /* Module heap types. */ PyTypeObject *isocalendar_date_type; @@ -164,12 +167,12 @@ _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); datetime_state *st = interp->datetime_module_state; - if (st != NULL && st->isocalendar_date_type != NULL) { - PyObject *mod = PyType_GetModule(st->isocalendar_date_type); - assert(mod != NULL); - *p_mod = Py_NewRef(mod); + if (st != NULL && st->module != NULL) { + assert(PyModule_CheckExact(st->module)); + *p_mod = Py_NewRef(st->module); return st; } + PyObject *mod = get_current_module(interp, NULL); if (mod == NULL) { assert(!PyErr_Occurred()); @@ -7227,6 +7230,8 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) static int init_state(datetime_state *st, PyObject *module, PyObject *old_module) { + st->module = Py_NewRef(module); + /* Each module gets its own heap types. */ #define ADD_TYPE(FIELD, SPEC, BASE) \ do { \ @@ -7245,6 +7250,7 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) assert(old_module != module); datetime_state *st_old = get_module_state(old_module); *st = (datetime_state){ + .module = st->module, .isocalendar_date_type = st->isocalendar_date_type, .us_per_ms = Py_NewRef(st_old->us_per_ms), .us_per_second = Py_NewRef(st_old->us_per_second), @@ -7304,7 +7310,7 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) static int traverse_state(datetime_state *st, visitproc visit, void *arg) { - /* heap types */ + Py_VISIT(st->module); Py_VISIT(st->isocalendar_date_type); return 0; @@ -7313,6 +7319,7 @@ traverse_state(datetime_state *st, visitproc visit, void *arg) static int clear_state(datetime_state *st) { + Py_CLEAR(st->module); /* Invalidate first */ Py_CLEAR(st->isocalendar_date_type); Py_CLEAR(st->us_per_ms); Py_CLEAR(st->us_per_second); From 2e1a1290a766aa300d6b50be45f38b22b5abe08d Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 24 Apr 2025 03:07:35 +0900 Subject: [PATCH 24/55] Comment --- Modules/_datetimemodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 8c62c073cc335e..5d34cb8638ed10 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -35,7 +35,8 @@ static PyTypeObject PyDateTime_TimeZoneType; typedef struct { - /* Corresponding module */ + /* Corresponding module that can be referred even after + * its weak reference stops working at shutdown. */ PyObject *module; /* Module heap types. */ From 3c059899552c57ac1a65c327ebffb527252fd4be Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 24 Apr 2025 11:15:46 +0900 Subject: [PATCH 25/55] assert(!_Py_IsInterpreterFinalizing()) --- Modules/_datetimemodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 5d34cb8638ed10..95f58d573efeff 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -12,6 +12,7 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() +#include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_time.h" // _PyTime_ObjectToTime_t() #include "pycore_unicodeobject.h" // _PyUnicode_Copy() @@ -180,6 +181,7 @@ _get_current_state(PyObject **p_mod) if (PyErr_Occurred()) { return NULL; } + assert(!_Py_IsInterpreterFinalizing(interp)); /* The static types can outlive the module, * so we must re-import the module. */ mod = PyImport_ImportModule("_datetime"); From 8eaae369fecfa5baca5fb9a69906b54770d1645a Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 24 Apr 2025 11:49:04 +0900 Subject: [PATCH 26/55] assign NULL to interp-state on error --- Modules/_datetimemodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 95f58d573efeff..3adfbab9a3825f 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -213,9 +213,7 @@ set_current_module(PyInterpreterState *interp, PyObject *mod) } int rc = PyDict_SetItem(dict, INTERP_KEY, ref); Py_DECREF(ref); - if (rc == 0) { - interp->datetime_module_state = get_module_state(mod); - } + interp->datetime_module_state = rc == 0 ? get_module_state(mod) : NULL; return rc; } From c2758bd610828c8b09a5ca8680398fbd4d74b70d Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Fri, 25 Apr 2025 12:16:57 +0900 Subject: [PATCH 27/55] Get module without interp-dict and weakref --- .../pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - .../internal/pycore_runtime_init_generated.h | 1 - .../internal/pycore_unicodeobject_generated.h | 4 - Modules/_datetimemodule.c | 115 +++--------------- 5 files changed, 18 insertions(+), 104 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 5485d0bd64f3f1..0203dd0e32ab7c 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -835,7 +835,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cafile)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 3ce192511e3879..14befd2128a9f7 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -326,7 +326,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(c_call) STRUCT_FOR_ID(c_exception) STRUCT_FOR_ID(c_return) - STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) STRUCT_FOR_ID(cafile) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 5c95d0feddecba..0c89372562df17 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -833,7 +833,6 @@ extern "C" { INIT_ID(c_call), \ INIT_ID(c_exception), \ INIT_ID(c_return), \ - INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ INIT_ID(cafile), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index a1fc9736d66618..25d71867d49ef5 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1092,10 +1092,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(cached_datetime_module); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_statements); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 3adfbab9a3825f..946f6f8421778e 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -36,8 +36,7 @@ static PyTypeObject PyDateTime_TimeZoneType; typedef struct { - /* Corresponding module that can be referred even after - * its weak reference stops working at shutdown. */ + /* Corresponding module that is referenced via the interpreter state. */ PyObject *module; /* Module heap types. */ @@ -126,68 +125,37 @@ get_module_state(PyObject *module) } -#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) - static PyObject * get_current_module(PyInterpreterState *interp, int *p_reloading) { - PyObject *mod = NULL; - int reloading = 0; - - PyObject *dict = PyInterpreterState_GetDict(interp); - if (dict == NULL) { - goto error; - } - PyObject *ref = NULL; - if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { - goto error; - } - if (ref != NULL) { - reloading = 1; - if (ref != Py_None) { - (void)PyWeakref_GetRef(ref, &mod); - if (mod == Py_None) { - Py_CLEAR(mod); - } - Py_DECREF(ref); - } - } + datetime_state *st = interp->datetime_module_state; if (p_reloading != NULL) { - *p_reloading = reloading; + *p_reloading = st != NULL ? 1 : 0; + } + if (st != NULL && st != (void *)Py_None && st->module != NULL) { + assert(PyModule_CheckExact(st->module)); + return Py_NewRef(st->module); } - return mod; - -error: - assert(PyErr_Occurred()); return NULL; } -static PyModuleDef datetimemodule; - static datetime_state * _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); datetime_state *st = interp->datetime_module_state; - if (st != NULL && st->module != NULL) { + if (st != NULL && st != (void *)Py_None && st->module != NULL) { assert(PyModule_CheckExact(st->module)); *p_mod = Py_NewRef(st->module); return st; } - PyObject *mod = get_current_module(interp, NULL); + assert(!_Py_IsInterpreterFinalizing(interp)); + /* The static types can outlive the module, + * so we must re-import the module. */ + PyObject *mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { - assert(!PyErr_Occurred()); - if (PyErr_Occurred()) { - return NULL; - } - assert(!_Py_IsInterpreterFinalizing(interp)); - /* The static types can outlive the module, - * so we must re-import the module. */ - mod = PyImport_ImportModule("_datetime"); - if (mod == NULL) { - return NULL; - } + return NULL; } st = get_module_state(mod); *p_mod = mod; @@ -203,65 +171,18 @@ static int set_current_module(PyInterpreterState *interp, PyObject *mod) { assert(mod != NULL); - PyObject *dict = PyInterpreterState_GetDict(interp); - if (dict == NULL) { - return -1; - } - PyObject *ref = PyWeakref_NewRef(mod, NULL); - if (ref == NULL) { - return -1; - } - int rc = PyDict_SetItem(dict, INTERP_KEY, ref); - Py_DECREF(ref); - interp->datetime_module_state = rc == 0 ? get_module_state(mod) : NULL; - return rc; + interp->datetime_module_state = get_module_state(mod); + return 0; } static void clear_current_module(PyInterpreterState *interp, PyObject *expected) { - PyObject *exc = PyErr_GetRaisedException(); - - PyObject *dict = PyInterpreterState_GetDict(interp); - if (dict == NULL) { - goto error; - } - - if (expected != NULL) { - PyObject *ref = NULL; - if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { - goto error; - } - if (ref != NULL) { - PyObject *current = NULL; - int rc = PyWeakref_GetRef(ref, ¤t); - /* We only need "current" for pointer comparison. */ - Py_XDECREF(current); - Py_DECREF(ref); - if (rc < 0) { - goto error; - } - if (current != expected) { - goto finally; - } - } + if (expected && get_module_state(expected) != interp->datetime_module_state) { + return; } - /* We use None to identify that the module was previously loaded. */ - if (PyDict_SetItem(dict, INTERP_KEY, Py_None) < 0) { - goto error; - } - - goto finally; - -error: - PyErr_FormatUnraisable("Exception ignored while clearing _datetime module"); - -finally: - if (!expected || get_module_state(expected) == interp->datetime_module_state) { - interp->datetime_module_state = NULL; - } - PyErr_SetRaisedException(exc); + interp->datetime_module_state = Py_None; } From 56f8a067fda35e61b9364e6a2d0c9de00dfdd1d3 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sat, 26 Apr 2025 03:31:58 +0900 Subject: [PATCH 28/55] Add tests --- Lib/test/datetimetester.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 898389ef409c05..7d03beb1dccc68 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7294,6 +7294,32 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) + def test_module_free(self): + script = textwrap.dedent(""" + import sys + import gc + import weakref + ws = weakref.WeakSet() + for _ in range(3): + import _datetime + ws.add(_datetime) + del sys.modules["_datetime"] + del _datetime + gc.collect() + assert len(ws) == 0 + """) + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) + + @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") + def test_no_leak(self): + script = textwrap.dedent(""" + import datetime + datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') + """) + res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) + self.assertIn(b'[0 refs, 0 blocks]', res.err) + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) From aa868072bb94cd8839067c5be398d4c25d352d35 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sat, 26 Apr 2025 07:40:39 +0900 Subject: [PATCH 29/55] Add tests --- Lib/test/datetimetester.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 7d03beb1dccc68..73d9804a28d829 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7294,6 +7294,43 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) + def test_module_state_at_shutdown2(self): + script = textwrap.dedent(""" + import sys + import _datetime + timedelta = _datetime.timedelta + del _datetime + del sys.modules["_datetime"] + + def gen(): + try: + yield + finally: + td = timedelta(days=1) # crash + assert td.days == 1 + assert not sys.modules + + it = gen() + next(it) + """) + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) + + def test_module_state_after_gc(self): + script = textwrap.dedent(""" + import sys + import gc + import _datetime + timedelta = _datetime.timedelta + del sys.modules['_datetime'] + del _datetime + gc.collect() + timedelta(days=1) + assert '_datetime' in sys.modules + """) + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) + def test_module_free(self): script = textwrap.dedent(""" import sys From 6880f34529530164518823715a42cac3b9e76ecf Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sat, 26 Apr 2025 19:53:58 +0900 Subject: [PATCH 30/55] Update and pass tests --- Lib/test/datetimetester.py | 44 +++++++++++++++++++++++++++----------- Modules/_datetimemodule.c | 24 ++++++++++++++++++++- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 73d9804a28d829..20df2edb061103 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7273,7 +7273,7 @@ def test_update_type_cache(self): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) - def test_module_state_at_shutdown(self): + def test_static_type_at_shutdown1(self): # gh-132413 script = textwrap.dedent(""" import sys @@ -7284,7 +7284,7 @@ def gen(): yield finally: assert not sys.modules - td = _datetime.timedelta(days=1) # crash + td = _datetime.timedelta(days=1) assert td.days == 1 assert not sys.modules @@ -7294,19 +7294,17 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) - def test_module_state_at_shutdown2(self): + def test_static_type_at_shutdown2(self): script = textwrap.dedent(""" import sys - import _datetime - timedelta = _datetime.timedelta - del _datetime - del sys.modules["_datetime"] + from _datetime import timedelta def gen(): try: yield finally: - td = timedelta(days=1) # crash + assert not sys.modules + td = timedelta(days=1) assert td.days == 1 assert not sys.modules @@ -7316,15 +7314,34 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) - def test_module_state_after_gc(self): + def test_static_type_at_shutdown3(self): script = textwrap.dedent(""" + import gc import sys + from _datetime import timedelta + del sys.modules['_datetime'] + gc.collect() + + def gen(): + try: + yield + finally: + timedelta(days=1) + + it = gen() + next(it) + """) + res = script_helper.assert_python_ok('-c', script) + self.assertIn(b'ImportError: sys.meta_path is None', res.err) + + def test_static_type_before_shutdown(self): + script = textwrap.dedent(""" import gc - import _datetime - timedelta = _datetime.timedelta + import sys + from _datetime import timedelta del sys.modules['_datetime'] - del _datetime gc.collect() + timedelta(days=1) assert '_datetime' in sys.modules """) @@ -7339,8 +7356,9 @@ def test_module_free(self): ws = weakref.WeakSet() for _ in range(3): import _datetime + td = _datetime.timedelta ws.add(_datetime) - del sys.modules["_datetime"] + del sys.modules['_datetime'] del _datetime gc.collect() assert len(ws) == 0 diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 946f6f8421778e..9acda8238b67de 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -150,11 +150,12 @@ _get_current_state(PyObject **p_mod) return st; } - assert(!_Py_IsInterpreterFinalizing(interp)); /* The static types can outlive the module, * so we must re-import the module. */ PyObject *mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { + assert(_Py_IsInterpreterFinalizing(interp)); + /* It is not preferable to reload the module implicitly here. */ return NULL; } st = get_module_state(mod); @@ -2047,6 +2048,9 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) @@ -2126,6 +2130,9 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { @@ -2711,6 +2718,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } /* Argument objects. */ PyObject *day = NULL; @@ -2930,6 +2940,9 @@ delta_total_seconds(PyObject *op, PyObject *Py_UNUSED(dummy)) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); @@ -3713,6 +3726,9 @@ date_isocalendar(PyObject *self, PyObject *Py_UNUSED(dummy)) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), year, week + 1, day + 1); @@ -6538,6 +6554,9 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); RELEASE_CURRENT_STATE(st, current_mod); @@ -6782,6 +6801,9 @@ datetime_timestamp(PyObject *op, PyObject *Py_UNUSED(dummy)) if (HASTZINFO(self) && self->tzinfo != Py_None) { PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (current_mod == NULL) { + return NULL; + } PyObject *delta; delta = datetime_subtract(op, CONST_EPOCH(st)); From 022d4e8148bf1fdbcce9556b39d53a2547cb9607 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 27 Apr 2025 15:46:05 +0900 Subject: [PATCH 31/55] Make tests realistic by using subinterp --- Lib/test/datetimetester.py | 80 +++++++++++++++++++++++++++--------- Modules/_datetimemodule.c | 2 +- Modules/_testcapi/datetime.c | 11 +++-- 3 files changed, 69 insertions(+), 24 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 20df2edb061103..0438ec42a7eae4 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7176,6 +7176,7 @@ def test_type_check_in_subinterp(self): spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + module.test_datetime_capi() def run(type_checker, obj): if not type_checker(obj, True): @@ -7314,38 +7315,77 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) + def run_script_132413(self, script): + # iOS requires the use of the custom framework loader, + # not the ExtensionFileLoader. + if sys.platform == "ios": + extension_loader = "AppleFrameworkLoader" + else: + extension_loader = "ExtensionFileLoader" + + main_interp_script = textwrap.dedent(f''' + import textwrap + from test import support + + sub_script = textwrap.dedent(""" + if {_interpreters is None}: + import _testcapi as module + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # Skip calling test_datetime_capi() + $REPLACE_ME$ + """) + + import _testcapi + _testcapi.test_datetime_capi() + + if {_interpreters is None}: + ret = support.run_in_subinterp(sub_script) + else: + import _interpreters + config = _interpreters.new_config('isolated').__dict__ + ret = support.run_in_subinterp_with_config(sub_script, **config) + + assert ret == 0 + + ''').replace('$REPLACE_ME$', textwrap.indent(script, '\x20'*4)) + + res = script_helper.assert_python_ok('-c', main_interp_script) + return res + def test_static_type_at_shutdown3(self): script = textwrap.dedent(""" - import gc - import sys - from _datetime import timedelta - del sys.modules['_datetime'] - gc.collect() + timedelta = module.get_delta_type() def gen(): - try: - yield - finally: - timedelta(days=1) + try: + yield + finally: + timedelta(days=1) it = gen() next(it) - """) - res = script_helper.assert_python_ok('-c', script) + """) + res = self.run_script_132413(script) self.assertIn(b'ImportError: sys.meta_path is None', res.err) def test_static_type_before_shutdown(self): - script = textwrap.dedent(""" - import gc + script = textwrap.dedent(f""" import sys - from _datetime import timedelta - del sys.modules['_datetime'] - gc.collect() - + assert '_datetime' not in sys.modules + timedelta = module.get_delta_type() timedelta(days=1) assert '_datetime' in sys.modules - """) - res = script_helper.assert_python_ok('-c', script) + """) + res = self.run_script_132413(script) self.assertFalse(res.err) def test_module_free(self): @@ -7356,7 +7396,7 @@ def test_module_free(self): ws = weakref.WeakSet() for _ in range(3): import _datetime - td = _datetime.timedelta + timedelta = _datetime.timedelta ws.add(_datetime) del sys.modules['_datetime'] del _datetime diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9acda8238b67de..947bb901532c71 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -155,7 +155,7 @@ _get_current_state(PyObject **p_mod) PyObject *mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { assert(_Py_IsInterpreterFinalizing(interp)); - /* It is not preferable to reload the module implicitly here. */ + /* We do not take care of the unlikely case. */ return NULL; } st = get_module_state(mod); diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index b800f9b8eb3473..daf0b8140a382a 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -453,6 +453,12 @@ test_PyDateTime_DELTA_GET(PyObject *self, PyObject *obj) return Py_BuildValue("(iii)", days, seconds, microseconds); } +static PyObject * +get_delta_type(PyObject *self, PyObject *args) +{ + return PyDateTimeAPI ? Py_NewRef(PyDateTimeAPI->DeltaType) : Py_None; +} + static PyMethodDef test_methods[] = { {"PyDateTime_DATE_GET", test_PyDateTime_DATE_GET, METH_O}, {"PyDateTime_DELTA_GET", test_PyDateTime_DELTA_GET, METH_O}, @@ -469,6 +475,7 @@ static PyMethodDef test_methods[] = { {"get_datetime_fromdateandtimeandfold", get_datetime_fromdateandtimeandfold, METH_VARARGS}, {"get_datetime_fromtimestamp", get_datetime_fromtimestamp, METH_VARARGS}, {"get_delta_fromdsu", get_delta_fromdsu, METH_VARARGS}, + {"get_delta_type", get_delta_type, METH_NOARGS}, {"get_time_fromtime", get_time_fromtime, METH_VARARGS}, {"get_time_fromtimeandfold", get_time_fromtimeandfold, METH_VARARGS}, {"get_timezone_utc_capi", get_timezone_utc_capi, METH_VARARGS}, @@ -495,9 +502,7 @@ _PyTestCapi_Init_DateTime(PyObject *mod) static int _testcapi_datetime_exec(PyObject *mod) { - if (test_datetime_capi(NULL, NULL) == NULL) { - return -1; - } + // Call test_datetime_capi() in each test. return 0; } From 318888ad07f4e4324bf07b8efee105aa093958c3 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 27 Apr 2025 16:24:12 +0900 Subject: [PATCH 32/55] Nit --- Lib/test/datetimetester.py | 16 ++++++++-------- Modules/_datetimemodule.c | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 0438ec42a7eae4..9a583f97df5bfd 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7291,7 +7291,7 @@ def gen(): it = gen() next(it) - """) + """) res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) @@ -7311,7 +7311,7 @@ def gen(): it = gen() next(it) - """) + """) res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) @@ -7366,10 +7366,10 @@ def test_static_type_at_shutdown3(self): timedelta = module.get_delta_type() def gen(): - try: - yield - finally: - timedelta(days=1) + try: + yield + finally: + timedelta(days=1) it = gen() next(it) @@ -7402,7 +7402,7 @@ def test_module_free(self): del _datetime gc.collect() assert len(ws) == 0 - """) + """) res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) @@ -7411,7 +7411,7 @@ def test_no_leak(self): script = textwrap.dedent(""" import datetime datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') - """) + """) res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) self.assertIn(b'[0 refs, 0 blocks]', res.err) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 947bb901532c71..cad082a1521ac1 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -155,7 +155,6 @@ _get_current_state(PyObject **p_mod) PyObject *mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { assert(_Py_IsInterpreterFinalizing(interp)); - /* We do not take care of the unlikely case. */ return NULL; } st = get_module_state(mod); From 6e6c3288d5d7cd9b7e116ade9d756f9651b40c01 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 28 Apr 2025 02:44:01 +0900 Subject: [PATCH 33/55] Make tests generic --- Lib/test/datetimetester.py | 139 +++++++++++++++-------------------- Modules/_testcapi/datetime.c | 31 +++++++- 2 files changed, 87 insertions(+), 83 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 9a583f97df5bfd..825aad21ec5023 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7155,7 +7155,10 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) - def test_type_check_in_subinterp(self): + def assert_python_ok_in_subinterp(self, script, + setup='_testcapi.test_datetime_capi()', + mainsetup='_testcapi.test_datetime_capi()', + config='isolated'): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. if sys.platform == "ios": @@ -7163,41 +7166,64 @@ def test_type_check_in_subinterp(self): else: extension_loader = "ExtensionFileLoader" - script = textwrap.dedent(f""" + maincode = textwrap.dedent(f''' + import textwrap + from test import support + + subcode = textwrap.dedent(""" + if {_interpreters is None}: + import _testcapi + else: + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + _testcapi = importlib.util.module_from_spec(spec) + spec.loader.exec_module(_testcapi) + + $SETUP$ + $SCRIPT$ + """) + + import _testcapi + $MAINSETUP$ + if {_interpreters is None}: - import _testcapi as module - module.test_datetime_capi() + ret = support.run_in_subinterp(subcode) else: - import importlib.machinery - import importlib.util - fullname = '_testcapi_datetime' - origin = importlib.util.find_spec('_testcapi').origin - loader = importlib.machinery.{extension_loader}(fullname, origin) - spec = importlib.util.spec_from_loader(fullname, loader) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - module.test_datetime_capi() + import _interpreters + config = _interpreters.new_config('{config}').__dict__ + ret = support.run_in_subinterp_with_config(subcode, **config) + + assert ret == 0 + + ''').replace('$MAINSETUP$', mainsetup) + maincode = maincode.replace('$SETUP$', textwrap.indent(setup, '\x20'*4)) + maincode = maincode.replace('$SCRIPT$', textwrap.indent(script, '\x20'*4)) + + res = script_helper.assert_python_ok('-c', maincode) + return res + def test_type_check_in_subinterp(self): + script = textwrap.dedent(f""" def run(type_checker, obj): if not type_checker(obj, True): raise TypeError(f'{{type(obj)}} is not C API type') import _datetime - run(module.datetime_check_date, _datetime.date.today()) - run(module.datetime_check_datetime, _datetime.datetime.now()) - run(module.datetime_check_time, _datetime.time(12, 30)) - run(module.datetime_check_delta, _datetime.timedelta(1)) - run(module.datetime_check_tzinfo, _datetime.tzinfo()) + run(_testcapi.datetime_check_date, _datetime.date.today()) + run(_testcapi.datetime_check_datetime, _datetime.datetime.now()) + run(_testcapi.datetime_check_time, _datetime.time(12, 30)) + run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) + run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) """) - if _interpreters is None: - ret = support.run_in_subinterp(script) - self.assertEqual(ret, 0) - else: - for name in ('isolated', 'legacy'): - with self.subTest(name): - config = _interpreters.new_config(name).__dict__ - ret = support.run_in_subinterp_with_config(script, **config) - self.assertEqual(ret, 0) + self.assert_python_ok_in_subinterp(script, mainsetup='') + if _interpreters is not None: + with self.subTest('legacy'): + self.assert_python_ok_in_subinterp(script, mainsetup='', + config='legacy') class ExtensionModuleTests(unittest.TestCase): @@ -7284,6 +7310,7 @@ def gen(): try: yield finally: + # Exceptions are ignored here assert not sys.modules td = _datetime.timedelta(days=1) assert td.days == 1 @@ -7315,55 +7342,9 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) - def run_script_132413(self, script): - # iOS requires the use of the custom framework loader, - # not the ExtensionFileLoader. - if sys.platform == "ios": - extension_loader = "AppleFrameworkLoader" - else: - extension_loader = "ExtensionFileLoader" - - main_interp_script = textwrap.dedent(f''' - import textwrap - from test import support - - sub_script = textwrap.dedent(""" - if {_interpreters is None}: - import _testcapi as module - else: - import importlib.machinery - import importlib.util - fullname = '_testcapi_datetime' - origin = importlib.util.find_spec('_testcapi').origin - loader = importlib.machinery.{extension_loader}(fullname, origin) - spec = importlib.util.spec_from_loader(fullname, loader) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - # Skip calling test_datetime_capi() - $REPLACE_ME$ - """) - - import _testcapi - _testcapi.test_datetime_capi() - - if {_interpreters is None}: - ret = support.run_in_subinterp(sub_script) - else: - import _interpreters - config = _interpreters.new_config('isolated').__dict__ - ret = support.run_in_subinterp_with_config(sub_script, **config) - - assert ret == 0 - - ''').replace('$REPLACE_ME$', textwrap.indent(script, '\x20'*4)) - - res = script_helper.assert_python_ok('-c', main_interp_script) - return res - def test_static_type_at_shutdown3(self): script = textwrap.dedent(""" - timedelta = module.get_delta_type() + timedelta = _testcapi.get_capi_types()['timedelta'] def gen(): try: @@ -7374,19 +7355,18 @@ def gen(): it = gen() next(it) """) - res = self.run_script_132413(script) + res = CapiTest.assert_python_ok_in_subinterp(self, script, setup='') self.assertIn(b'ImportError: sys.meta_path is None', res.err) def test_static_type_before_shutdown(self): script = textwrap.dedent(f""" import sys assert '_datetime' not in sys.modules - timedelta = module.get_delta_type() + timedelta = _testcapi.get_capi_types()['timedelta'] timedelta(days=1) assert '_datetime' in sys.modules """) - res = self.run_script_132413(script) - self.assertFalse(res.err) + CapiTest.assert_python_ok_in_subinterp(self, script, setup='') def test_module_free(self): script = textwrap.dedent(""" @@ -7403,8 +7383,7 @@ def test_module_free(self): gc.collect() assert len(ws) == 0 """) - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) + script_helper.assert_python_ok('-c', script) @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") def test_no_leak(self): diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index daf0b8140a382a..0b4a968d0a0d0c 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -454,9 +454,34 @@ test_PyDateTime_DELTA_GET(PyObject *self, PyObject *obj) } static PyObject * -get_delta_type(PyObject *self, PyObject *args) +get_capi_types(PyObject *self, PyObject *args) { - return PyDateTimeAPI ? Py_NewRef(PyDateTimeAPI->DeltaType) : Py_None; + if (PyDateTimeAPI == NULL) { + Py_RETURN_NONE; + } + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + if (PyDict_SetItemString(dict, "date", (PyObject *)PyDateTimeAPI->DateType) < 0) { + goto error; + } + if (PyDict_SetItemString(dict, "time", (PyObject *)PyDateTimeAPI->TimeType) < 0) { + goto error; + } + if (PyDict_SetItemString(dict, "datetime", (PyObject *)PyDateTimeAPI->DateTimeType) < 0) { + goto error; + } + if (PyDict_SetItemString(dict, "timedelta", (PyObject *)PyDateTimeAPI->DeltaType) < 0) { + goto error; + } + if (PyDict_SetItemString(dict, "tzinfo", (PyObject *)PyDateTimeAPI->TZInfoType) < 0) { + goto error; + } + return dict; +error: + Py_DECREF(dict); + return NULL; } static PyMethodDef test_methods[] = { @@ -475,11 +500,11 @@ static PyMethodDef test_methods[] = { {"get_datetime_fromdateandtimeandfold", get_datetime_fromdateandtimeandfold, METH_VARARGS}, {"get_datetime_fromtimestamp", get_datetime_fromtimestamp, METH_VARARGS}, {"get_delta_fromdsu", get_delta_fromdsu, METH_VARARGS}, - {"get_delta_type", get_delta_type, METH_NOARGS}, {"get_time_fromtime", get_time_fromtime, METH_VARARGS}, {"get_time_fromtimeandfold", get_time_fromtimeandfold, METH_VARARGS}, {"get_timezone_utc_capi", get_timezone_utc_capi, METH_VARARGS}, {"get_timezones_offset_zero", get_timezones_offset_zero, METH_NOARGS}, + {"get_capi_types", get_capi_types, METH_NOARGS}, {"make_timezones_capi", make_timezones_capi, METH_NOARGS}, {"test_datetime_capi", test_datetime_capi, METH_NOARGS}, {NULL}, From 351225e9ac0eb1818edfedccc331bfb6165b7e94 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 28 Apr 2025 07:56:39 +0900 Subject: [PATCH 34/55] Fix test_datetime_capi() for subinterps --- Lib/test/datetimetester.py | 40 ++++++++++++++++++++++-------------- Modules/_testcapi/datetime.c | 2 +- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 825aad21ec5023..1b1f7b57446132 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7157,7 +7157,6 @@ def test_datetime_from_timestamp(self): def assert_python_ok_in_subinterp(self, script, setup='_testcapi.test_datetime_capi()', - mainsetup='_testcapi.test_datetime_capi()', config='isolated'): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. @@ -7166,7 +7165,7 @@ def assert_python_ok_in_subinterp(self, script, else: extension_loader = "ExtensionFileLoader" - maincode = textwrap.dedent(f''' + code = textwrap.dedent(f''' import textwrap from test import support @@ -7183,12 +7182,11 @@ def assert_python_ok_in_subinterp(self, script, _testcapi = importlib.util.module_from_spec(spec) spec.loader.exec_module(_testcapi) - $SETUP$ $SCRIPT$ """) import _testcapi - $MAINSETUP$ + $SETUP$ if {_interpreters is None}: ret = support.run_in_subinterp(subcode) @@ -7199,11 +7197,11 @@ def assert_python_ok_in_subinterp(self, script, assert ret == 0 - ''').replace('$MAINSETUP$', mainsetup) - maincode = maincode.replace('$SETUP$', textwrap.indent(setup, '\x20'*4)) - maincode = maincode.replace('$SCRIPT$', textwrap.indent(script, '\x20'*4)) + ''').rstrip() + code = code.replace('$SETUP$', setup) + code = code.replace('$SCRIPT$', textwrap.indent(script, '\x20'*4)) - res = script_helper.assert_python_ok('-c', maincode) + res = script_helper.assert_python_ok('-c', code) return res def test_type_check_in_subinterp(self): @@ -7212,6 +7210,7 @@ def run(type_checker, obj): if not type_checker(obj, True): raise TypeError(f'{{type(obj)}} is not C API type') + _testcapi.test_datetime_capi() import _datetime run(_testcapi.datetime_check_date, _datetime.date.today()) run(_testcapi.datetime_check_datetime, _datetime.datetime.now()) @@ -7219,11 +7218,10 @@ def run(type_checker, obj): run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) """) - self.assert_python_ok_in_subinterp(script, mainsetup='') + self.assert_python_ok_in_subinterp(script, '') if _interpreters is not None: - with self.subTest('legacy'): - self.assert_python_ok_in_subinterp(script, mainsetup='', - config='legacy') + with self.subTest(name := 'legacy'): + self.assert_python_ok_in_subinterp(script, '', name) class ExtensionModuleTests(unittest.TestCase): @@ -7355,8 +7353,20 @@ def gen(): it = gen() next(it) """) - res = CapiTest.assert_python_ok_in_subinterp(self, script, setup='') - self.assertIn(b'ImportError: sys.meta_path is None', res.err) + + with self.subTest('PyDateTime_IMPORT by MainInterpreter'): + res = CapiTest.assert_python_ok_in_subinterp(self, script) + self.assertIn(b'ImportError: sys.meta_path is None', res.err) + + script2 = f'_testcapi.test_datetime_capi()\n{script}' + + with self.subTest('PyDateTime_IMPORT by Subinterpreter'): + res = CapiTest.assert_python_ok_in_subinterp(self, script2, '') + self.assertFalse(res.err) + + with self.subTest('PyDateTime_IMPORT by Main/Sub'): + res = CapiTest.assert_python_ok_in_subinterp(self, script2) + self.assertFalse(res.err) def test_static_type_before_shutdown(self): script = textwrap.dedent(f""" @@ -7366,7 +7376,7 @@ def test_static_type_before_shutdown(self): timedelta(days=1) assert '_datetime' in sys.modules """) - CapiTest.assert_python_ok_in_subinterp(self, script, setup='') + CapiTest.assert_python_ok_in_subinterp(self, script) def test_module_free(self): script = textwrap.dedent(""" diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 0b4a968d0a0d0c..4ab18382c72a88 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -11,7 +11,7 @@ test_datetime_capi(PyObject *self, PyObject *args) if (PyDateTimeAPI) { if (test_run_counter) { /* Probably regrtest.py -R */ - Py_RETURN_NONE; + // Interpreters need their module, so call PyDateTime_IMPORT } else { PyErr_SetString(PyExc_AssertionError, From 419500eec2bbb62027261b175f9a5ad99f518807 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 28 Apr 2025 19:54:56 +0900 Subject: [PATCH 35/55] Use assert_python_failure() --- Lib/test/datetimetester.py | 39 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 1b1f7b57446132..3447ec86375d9f 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7155,9 +7155,9 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) - def assert_python_ok_in_subinterp(self, script, - setup='_testcapi.test_datetime_capi()', - config='isolated'): + def assert_python_in_subinterp(self, check_if_ok: bool, script, + setup='_testcapi.test_datetime_capi()', + config='isolated'): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. if sys.platform == "ios": @@ -7201,7 +7201,10 @@ def assert_python_ok_in_subinterp(self, script, code = code.replace('$SETUP$', setup) code = code.replace('$SCRIPT$', textwrap.indent(script, '\x20'*4)) - res = script_helper.assert_python_ok('-c', code) + if check_if_ok: + res = script_helper.assert_python_ok('-c', code) + else: + res = script_helper.assert_python_failure('-c', code) return res def test_type_check_in_subinterp(self): @@ -7217,11 +7220,11 @@ def run(type_checker, obj): run(_testcapi.datetime_check_time, _datetime.time(12, 30)) run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) - """) - self.assert_python_ok_in_subinterp(script, '') + """) + self.assert_python_in_subinterp(True, script, '') if _interpreters is not None: with self.subTest(name := 'legacy'): - self.assert_python_ok_in_subinterp(script, '', name) + self.assert_python_in_subinterp(True, script, '', name) class ExtensionModuleTests(unittest.TestCase): @@ -7316,7 +7319,7 @@ def gen(): it = gen() next(it) - """) + """) res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) @@ -7336,7 +7339,7 @@ def gen(): it = gen() next(it) - """) + """) res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) @@ -7352,20 +7355,18 @@ def gen(): it = gen() next(it) - """) - + """) with self.subTest('PyDateTime_IMPORT by MainInterpreter'): - res = CapiTest.assert_python_ok_in_subinterp(self, script) + res = CapiTest.assert_python_in_subinterp(self, True, script) self.assertIn(b'ImportError: sys.meta_path is None', res.err) script2 = f'_testcapi.test_datetime_capi()\n{script}' - with self.subTest('PyDateTime_IMPORT by Subinterpreter'): - res = CapiTest.assert_python_ok_in_subinterp(self, script2, '') + res = CapiTest.assert_python_in_subinterp(self, True, script2, '') self.assertFalse(res.err) with self.subTest('PyDateTime_IMPORT by Main/Sub'): - res = CapiTest.assert_python_ok_in_subinterp(self, script2) + res = CapiTest.assert_python_in_subinterp(self, True, script2, '') self.assertFalse(res.err) def test_static_type_before_shutdown(self): @@ -7375,8 +7376,8 @@ def test_static_type_before_shutdown(self): timedelta = _testcapi.get_capi_types()['timedelta'] timedelta(days=1) assert '_datetime' in sys.modules - """) - CapiTest.assert_python_ok_in_subinterp(self, script) + """) + CapiTest.assert_python_in_subinterp(self, True, script) def test_module_free(self): script = textwrap.dedent(""" @@ -7392,7 +7393,7 @@ def test_module_free(self): del _datetime gc.collect() assert len(ws) == 0 - """) + """) script_helper.assert_python_ok('-c', script) @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") @@ -7400,7 +7401,7 @@ def test_no_leak(self): script = textwrap.dedent(""" import datetime datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') - """) + """) res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) self.assertIn(b'[0 refs, 0 blocks]', res.err) From 596007c00ce06c1d5761f580c49692d29289b9a2 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 28 Apr 2025 20:30:15 +0900 Subject: [PATCH 36/55] Add a failure test (crash) --- Lib/test/datetimetester.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 3447ec86375d9f..4f6668db4e4d96 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7301,6 +7301,26 @@ def test_update_type_cache(self): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) + def test_static_type_attr_on_subinterp(self): + script = textwrap.dedent(f""" + date = _testcapi.get_capi_types()['date'] + date.today + """) + # Fail before loaded + with self.subTest('[PyDateTime_IMPORT] main: yes sub: no'): + res = CapiTest.assert_python_in_subinterp(self, False, script) + self.assertIn(b'_PyType_CheckConsistency: Assertion failed', res.err) + self.assertIn(b'lookup_tp_dict(type) != ((void *)0)', res.err) + + # OK after loaded + with self.subTest('[PyDateTime_IMPORT] main: no sub: yes'): + script2 = f'_testcapi.test_datetime_capi()\n{script}' + CapiTest.assert_python_in_subinterp(self, True, script2) + + with self.subTest('Regular'): + script2 = f'import _datetime\n{script}' + CapiTest.assert_python_in_subinterp(self, True, script2) + def test_static_type_at_shutdown1(self): # gh-132413 script = textwrap.dedent(""" From 1dd9707405ec91aae1588aead94424c7d93a5d08 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Mon, 28 Apr 2025 21:12:51 +0900 Subject: [PATCH 37/55] Correct the previous commit --- Lib/test/datetimetester.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 4f6668db4e4d96..547fe885f12cf3 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7307,20 +7307,21 @@ def test_static_type_attr_on_subinterp(self): date.today """) # Fail before loaded - with self.subTest('[PyDateTime_IMPORT] main: yes sub: no'): - res = CapiTest.assert_python_in_subinterp(self, False, script) - self.assertIn(b'_PyType_CheckConsistency: Assertion failed', res.err) - self.assertIn(b'lookup_tp_dict(type) != ((void *)0)', res.err) + with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): + CapiTest.assert_python_in_subinterp(self, False, script) # OK after loaded - with self.subTest('[PyDateTime_IMPORT] main: no sub: yes'): - script2 = f'_testcapi.test_datetime_capi()\n{script}' - CapiTest.assert_python_in_subinterp(self, True, script2) + script2 = f'_testcapi.test_datetime_capi()\n{script}' + with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): + CapiTest.assert_python_in_subinterp(self, True, script2, '') - with self.subTest('Regular'): - script2 = f'import _datetime\n{script}' + with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): CapiTest.assert_python_in_subinterp(self, True, script2) + script3 = f'import _datetime\n{script}' + with self.subTest('Regular import'): + CapiTest.assert_python_in_subinterp(self, True, script3) + def test_static_type_at_shutdown1(self): # gh-132413 script = textwrap.dedent(""" From fb35703cc11fc6fa7196465ebdfd34268a46cece Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 29 Apr 2025 03:00:49 +0900 Subject: [PATCH 38/55] Redirect to the CapiTest method --- Lib/test/datetimetester.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 547fe885f12cf3..9f824ca2306cbd 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7301,6 +7301,9 @@ def test_update_type_cache(self): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) + def assert_python_in_subinterp(self, *args, **kwargs): + return CapiTest.assert_python_in_subinterp(self, *args, **kwargs) + def test_static_type_attr_on_subinterp(self): script = textwrap.dedent(f""" date = _testcapi.get_capi_types()['date'] @@ -7308,19 +7311,20 @@ def test_static_type_attr_on_subinterp(self): """) # Fail before loaded with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - CapiTest.assert_python_in_subinterp(self, False, script) + self.assert_python_in_subinterp(False, script) # OK after loaded script2 = f'_testcapi.test_datetime_capi()\n{script}' with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - CapiTest.assert_python_in_subinterp(self, True, script2, '') + self.assert_python_in_subinterp(True, script2, setup='') with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): - CapiTest.assert_python_in_subinterp(self, True, script2) + # Check if each test_datetime_capi() calls PyDateTime_IMPORT + self.assert_python_in_subinterp(True, script2) script3 = f'import _datetime\n{script}' with self.subTest('Regular import'): - CapiTest.assert_python_in_subinterp(self, True, script3) + self.assert_python_in_subinterp(True, script3) def test_static_type_at_shutdown1(self): # gh-132413 @@ -7377,17 +7381,18 @@ def gen(): it = gen() next(it) """) - with self.subTest('PyDateTime_IMPORT by MainInterpreter'): - res = CapiTest.assert_python_in_subinterp(self, True, script) + with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): + res = self.assert_python_in_subinterp(True, script) self.assertIn(b'ImportError: sys.meta_path is None', res.err) script2 = f'_testcapi.test_datetime_capi()\n{script}' - with self.subTest('PyDateTime_IMPORT by Subinterpreter'): - res = CapiTest.assert_python_in_subinterp(self, True, script2, '') + with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): + res = self.assert_python_in_subinterp(True, script2, setup='') self.assertFalse(res.err) - with self.subTest('PyDateTime_IMPORT by Main/Sub'): - res = CapiTest.assert_python_in_subinterp(self, True, script2, '') + with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): + # Check if each test_datetime_capi() calls PyDateTime_IMPORT + res = self.assert_python_in_subinterp(True, script2) self.assertFalse(res.err) def test_static_type_before_shutdown(self): @@ -7398,7 +7403,7 @@ def test_static_type_before_shutdown(self): timedelta(days=1) assert '_datetime' in sys.modules """) - CapiTest.assert_python_in_subinterp(self, True, script) + self.assert_python_in_subinterp(True, script) def test_module_free(self): script = textwrap.dedent(""" From c6cc066d8bd2f283567b644f7091dc70255aad97 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 29 Apr 2025 03:02:57 +0900 Subject: [PATCH 39/55] Merge two tests (1) --- Lib/test/datetimetester.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 9f824ca2306cbd..a09cf6a1b2c5f2 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7304,7 +7304,7 @@ def test_update_type_cache(self): def assert_python_in_subinterp(self, *args, **kwargs): return CapiTest.assert_python_in_subinterp(self, *args, **kwargs) - def test_static_type_attr_on_subinterp(self): + def test_static_type_on_subinterp(self): script = textwrap.dedent(f""" date = _testcapi.get_capi_types()['date'] date.today @@ -7326,6 +7326,18 @@ def test_static_type_attr_on_subinterp(self): with self.subTest('Regular import'): self.assert_python_in_subinterp(True, script3) + script4 = textwrap.dedent(f""" + import sys + assert '_datetime' not in sys.modules + timedelta = _testcapi.get_capi_types()['timedelta'] + timedelta(days=1) + assert '_datetime' in sys.modules + date = _testcapi.get_capi_types()['date'] + date.today + """) + with self.subTest('Implicit import'): + self.assert_python_in_subinterp(True, script4) + def test_static_type_at_shutdown1(self): # gh-132413 script = textwrap.dedent(""" @@ -7395,16 +7407,6 @@ def gen(): res = self.assert_python_in_subinterp(True, script2) self.assertFalse(res.err) - def test_static_type_before_shutdown(self): - script = textwrap.dedent(f""" - import sys - assert '_datetime' not in sys.modules - timedelta = _testcapi.get_capi_types()['timedelta'] - timedelta(days=1) - assert '_datetime' in sys.modules - """) - self.assert_python_in_subinterp(True, script) - def test_module_free(self): script = textwrap.dedent(""" import sys From cc04c6f4bfc6489908839332d8b7782604f4c88e Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 29 Apr 2025 03:07:58 +0900 Subject: [PATCH 40/55] Merge two tests (2) --- Lib/test/datetimetester.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index a09cf6a1b2c5f2..070174e56db53a 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7338,11 +7338,12 @@ def test_static_type_on_subinterp(self): with self.subTest('Implicit import'): self.assert_python_in_subinterp(True, script4) - def test_static_type_at_shutdown1(self): + def test_static_type_at_shutdown(self): # gh-132413 script = textwrap.dedent(""" import sys import _datetime + timedelta = _datetime.timedelta def gen(): try: @@ -7352,24 +7353,6 @@ def gen(): assert not sys.modules td = _datetime.timedelta(days=1) assert td.days == 1 - assert not sys.modules - - it = gen() - next(it) - """) - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) - - def test_static_type_at_shutdown2(self): - script = textwrap.dedent(""" - import sys - from _datetime import timedelta - - def gen(): - try: - yield - finally: - assert not sys.modules td = timedelta(days=1) assert td.days == 1 assert not sys.modules @@ -7377,8 +7360,12 @@ def gen(): it = gen() next(it) """) - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) + with self.subTest('MainInterpreter'): + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) + with self.subTest('Subinterpreter'): + res = self.assert_python_in_subinterp(True, script, setup='') + self.assertFalse(res.err) def test_static_type_at_shutdown3(self): script = textwrap.dedent(""" From 56267bdc968c1de5ca84e541540bb8bafcde326c Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 29 Apr 2025 03:10:04 +0900 Subject: [PATCH 41/55] Merge two tests (3) --- Lib/test/datetimetester.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 070174e56db53a..e6eeec4144d6ef 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7367,15 +7367,19 @@ def gen(): res = self.assert_python_in_subinterp(True, script, setup='') self.assertFalse(res.err) - def test_static_type_at_shutdown3(self): script = textwrap.dedent(""" + import sys timedelta = _testcapi.get_capi_types()['timedelta'] def gen(): try: yield finally: - timedelta(days=1) + # Exceptions are ignored here + assert not sys.modules + td = timedelta(days=1) + assert td.days == 1 + assert not sys.modules it = gen() next(it) From 063f37dd26ea675ec249f2c9cbb391e259d72171 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 29 Apr 2025 03:11:09 +0900 Subject: [PATCH 42/55] Add test cases --- Lib/test/datetimetester.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index e6eeec4144d6ef..9b33f4f700efa5 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7367,6 +7367,40 @@ def gen(): res = self.assert_python_in_subinterp(True, script, setup='') self.assertFalse(res.err) + script = textwrap.dedent(""" + import gc + import sys + import _testcapi + + def emulate_interp_restart(): + del sys.modules['_datetime'] + try: + del sys.modules['datetime'] + except KeyError: + pass + gc.collect() # unload + + _testcapi.test_datetime_capi() # only once + timedelta = _testcapi.get_capi_types()['timedelta'] + emulate_interp_restart() + timedelta(days=1) + emulate_interp_restart() + + def gen(): + try: + yield + finally: + # Exceptions are ignored here + assert not sys.modules + timedelta(days=1) + + it = gen() + next(it) + """) + with self.subTest('MainInterpreter Restart'): + res = script_helper.assert_python_ok('-c', script) + self.assertIn(b'ImportError: sys.meta_path is None', res.err) + script = textwrap.dedent(""" import sys timedelta = _testcapi.get_capi_types()['timedelta'] From 992fd0cf6039d27275c3c82a65805a5cf6b68671 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 30 Apr 2025 09:24:48 +0900 Subject: [PATCH 43/55] Move up a few tests --- Lib/test/datetimetester.py | 56 +++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 9b33f4f700efa5..5b1b280b3a4221 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7233,6 +7233,9 @@ def setUp(self): if self.__class__.__name__.endswith('Pure'): self.skipTest('Not relevant in pure Python') + def assert_python_in_subinterp(self, *args, **kwargs): + return CapiTest.assert_python_in_subinterp(self, *args, **kwargs) + @support.cpython_only def test_gh_120161(self): with self.subTest('simple'): @@ -7301,8 +7304,31 @@ def test_update_type_cache(self): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) - def assert_python_in_subinterp(self, *args, **kwargs): - return CapiTest.assert_python_in_subinterp(self, *args, **kwargs) + def test_module_free(self): + script = textwrap.dedent(""" + import sys + import gc + import weakref + ws = weakref.WeakSet() + for _ in range(3): + import _datetime + timedelta = _datetime.timedelta # static type + ws.add(_datetime) + del sys.modules['_datetime'] + del _datetime + gc.collect() + assert len(ws) == 0 + """) + script_helper.assert_python_ok('-c', script) + + @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") + def test_no_leak(self): + script = textwrap.dedent(""" + import datetime + datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') + """) + res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) + self.assertIn(b'[0 refs, 0 blocks]', res.err) def test_static_type_on_subinterp(self): script = textwrap.dedent(f""" @@ -7432,32 +7458,6 @@ def gen(): res = self.assert_python_in_subinterp(True, script2) self.assertFalse(res.err) - def test_module_free(self): - script = textwrap.dedent(""" - import sys - import gc - import weakref - ws = weakref.WeakSet() - for _ in range(3): - import _datetime - timedelta = _datetime.timedelta - ws.add(_datetime) - del sys.modules['_datetime'] - del _datetime - gc.collect() - assert len(ws) == 0 - """) - script_helper.assert_python_ok('-c', script) - - @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") - def test_no_leak(self): - script = textwrap.dedent(""" - import datetime - datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') - """) - res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) - self.assertIn(b'[0 refs, 0 blocks]', res.err) - def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) From f0f8fa0894bfc1a984cb665fa3d1aca428b139b9 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 30 Apr 2025 09:30:26 +0900 Subject: [PATCH 44/55] Update tests --- Lib/test/datetimetester.py | 119 ++++++++++++++++------------------- Modules/_testcapi/datetime.c | 5 +- 2 files changed, 57 insertions(+), 67 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 5b1b280b3a4221..2deeec663e5e01 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7155,9 +7155,8 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) - def assert_python_in_subinterp(self, check_if_ok: bool, script, - setup='_testcapi.test_datetime_capi()', - config='isolated'): + def assert_python_in_subinterp(self, check_if_ok, script, init='', + fini='', repeat=1, config='isolated'): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. if sys.platform == "ios": @@ -7166,40 +7165,41 @@ def assert_python_in_subinterp(self, check_if_ok: bool, script, extension_loader = "ExtensionFileLoader" code = textwrap.dedent(f''' - import textwrap - from test import support - - subcode = textwrap.dedent(""" - if {_interpreters is None}: - import _testcapi - else: - import importlib.machinery - import importlib.util - fullname = '_testcapi_datetime' - origin = importlib.util.find_spec('_testcapi').origin - loader = importlib.machinery.{extension_loader}(fullname, origin) - spec = importlib.util.spec_from_loader(fullname, loader) - _testcapi = importlib.util.module_from_spec(spec) - spec.loader.exec_module(_testcapi) - - $SCRIPT$ - """) - - import _testcapi - $SETUP$ - + subinterp_code = """ if {_interpreters is None}: - ret = support.run_in_subinterp(subcode) + import _testcapi else: - import _interpreters - config = _interpreters.new_config('{config}').__dict__ - ret = support.run_in_subinterp_with_config(subcode, **config) + import importlib.machinery + import importlib.util + fullname = '_testcapi_datetime' + origin = importlib.util.find_spec('_testcapi').origin + loader = importlib.machinery.{extension_loader}(fullname, origin) + spec = importlib.util.spec_from_loader(fullname, loader) + _testcapi = importlib.util.module_from_spec(spec) + spec.loader.exec_module(_testcapi) + INDEX = $INDEX$ + setup = _testcapi.test_datetime_capi # call it if needed + $SCRIPT$ + """ - assert ret == 0 + import _testcapi + from test import support + setup = _testcapi.test_datetime_capi + $INIT$ - ''').rstrip() - code = code.replace('$SETUP$', setup) - code = code.replace('$SCRIPT$', textwrap.indent(script, '\x20'*4)) + for idx in range({repeat}): + subcode = subinterp_code.replace('$INDEX$', str(idx)) + if {_interpreters is None}: + ret = support.run_in_subinterp(subcode) + else: + import _interpreters + config = _interpreters.new_config('{config}').__dict__ + ret = support.run_in_subinterp_with_config(subcode, **config) + assert ret == 0 + $FINI$ + ''') + code = code.replace('$INIT$', init).replace('$FINI$', fini) + code = code.replace('$SCRIPT$', script) if check_if_ok: res = script_helper.assert_python_ok('-c', code) @@ -7213,7 +7213,7 @@ def run(type_checker, obj): if not type_checker(obj, True): raise TypeError(f'{{type(obj)}} is not C API type') - _testcapi.test_datetime_capi() + setup() import _datetime run(_testcapi.datetime_check_date, _datetime.date.today()) run(_testcapi.datetime_check_datetime, _datetime.datetime.now()) @@ -7221,10 +7221,10 @@ def run(type_checker, obj): run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) """) - self.assert_python_in_subinterp(True, script, '') + self.assert_python_in_subinterp(True, script) if _interpreters is not None: with self.subTest(name := 'legacy'): - self.assert_python_in_subinterp(True, script, '', name) + self.assert_python_in_subinterp(True, script, config=name) class ExtensionModuleTests(unittest.TestCase): @@ -7331,38 +7331,34 @@ def test_no_leak(self): self.assertIn(b'[0 refs, 0 blocks]', res.err) def test_static_type_on_subinterp(self): - script = textwrap.dedent(f""" + script = textwrap.dedent(""" date = _testcapi.get_capi_types()['date'] date.today """) - # Fail before loaded with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - self.assert_python_in_subinterp(False, script) + # FIXME: Segfault + self.assert_python_in_subinterp(False, script, 'setup()') - # OK after loaded - script2 = f'_testcapi.test_datetime_capi()\n{script}' + with_setup = 'setup()' + script with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - self.assert_python_in_subinterp(True, script2, setup='') + self.assert_python_in_subinterp(True, with_setup) with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): - # Check if each test_datetime_capi() calls PyDateTime_IMPORT - self.assert_python_in_subinterp(True, script2) + # Check if PyDateTime_IMPORT is invoked not only once + self.assert_python_in_subinterp(True, with_setup, 'setup()') + self.assert_python_in_subinterp(True, 'setup()', fini=with_setup) + self.assert_python_in_subinterp(True, with_setup, repeat=2) - script3 = f'import _datetime\n{script}' - with self.subTest('Regular import'): - self.assert_python_in_subinterp(True, script3) + with_import = 'import _datetime' + script + with self.subTest('Explicit import'): + self.assert_python_in_subinterp(True, with_import, 'setup()') - script4 = textwrap.dedent(f""" - import sys - assert '_datetime' not in sys.modules + with_import = textwrap.dedent(""" timedelta = _testcapi.get_capi_types()['timedelta'] timedelta(days=1) - assert '_datetime' in sys.modules - date = _testcapi.get_capi_types()['date'] - date.today - """) + """) + script with self.subTest('Implicit import'): - self.assert_python_in_subinterp(True, script4) + self.assert_python_in_subinterp(True, with_import, 'setup()') def test_static_type_at_shutdown(self): # gh-132413 @@ -7390,7 +7386,7 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) with self.subTest('Subinterpreter'): - res = self.assert_python_in_subinterp(True, script, setup='') + res = self.assert_python_in_subinterp(True, script) self.assertFalse(res.err) script = textwrap.dedent(""" @@ -7445,17 +7441,12 @@ def gen(): next(it) """) with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - res = self.assert_python_in_subinterp(True, script) + res = self.assert_python_in_subinterp(True, script, 'setup()') self.assertIn(b'ImportError: sys.meta_path is None', res.err) - script2 = f'_testcapi.test_datetime_capi()\n{script}' + with_import = 'setup()' + script with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - res = self.assert_python_in_subinterp(True, script2, setup='') - self.assertFalse(res.err) - - with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): - # Check if each test_datetime_capi() calls PyDateTime_IMPORT - res = self.assert_python_in_subinterp(True, script2) + res = self.assert_python_in_subinterp(True, with_import) self.assertFalse(res.err) diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 4ab18382c72a88..42bce301a51277 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -11,7 +11,6 @@ test_datetime_capi(PyObject *self, PyObject *args) if (PyDateTimeAPI) { if (test_run_counter) { /* Probably regrtest.py -R */ - // Interpreters need their module, so call PyDateTime_IMPORT } else { PyErr_SetString(PyExc_AssertionError, @@ -20,7 +19,7 @@ test_datetime_capi(PyObject *self, PyObject *args) } } test_run_counter++; - PyDateTime_IMPORT; + PyDateTime_IMPORT; // Ensure interpreters individually import a module if (PyDateTimeAPI == NULL) { return NULL; @@ -527,7 +526,7 @@ _PyTestCapi_Init_DateTime(PyObject *mod) static int _testcapi_datetime_exec(PyObject *mod) { - // Call test_datetime_capi() in each test. + // The execution does not invoke test_datetime_capi() return 0; } From 7f31245634aafe3e7042346afdff5e9f6dfcf106 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 30 Apr 2025 09:31:14 +0900 Subject: [PATCH 45/55] Move a repeat test to test_embed --- Lib/test/datetimetester.py | 34 ---------------------------------- Lib/test/test_embed.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 2deeec663e5e01..9f21a23524b2da 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7389,40 +7389,6 @@ def gen(): res = self.assert_python_in_subinterp(True, script) self.assertFalse(res.err) - script = textwrap.dedent(""" - import gc - import sys - import _testcapi - - def emulate_interp_restart(): - del sys.modules['_datetime'] - try: - del sys.modules['datetime'] - except KeyError: - pass - gc.collect() # unload - - _testcapi.test_datetime_capi() # only once - timedelta = _testcapi.get_capi_types()['timedelta'] - emulate_interp_restart() - timedelta(days=1) - emulate_interp_restart() - - def gen(): - try: - yield - finally: - # Exceptions are ignored here - assert not sys.modules - timedelta(days=1) - - it = gen() - next(it) - """) - with self.subTest('MainInterpreter Restart'): - res = script_helper.assert_python_ok('-c', script) - self.assertIn(b'ImportError: sys.meta_path is None', res.err) - script = textwrap.dedent(""" import sys timedelta = _testcapi.get_capi_types()['timedelta'] diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e06e684408ca6b..afe2c2286b67ad 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -440,6 +440,37 @@ def test_datetime_reset_strptime(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '20000101\n' * INIT_LOOPS) + def test_datetime_capi_at_shutdown(self): + # gh-120782: Test the case where PyDateTime_IMPORT is called only once + code = textwrap.dedent(""" + import sys + import _testcapi + if not _testcapi.get_capi_types(): + _testcapi.test_datetime_capi() + assert '_datetime' in sys.modules + else: + assert '_datetime' not in sys.modules + timedelta = _testcapi.get_capi_types()['timedelta'] + + def gen(): + try: + yield + finally: + assert not sys.modules + res = 1 + try: + timedelta(days=1) + except ImportError as e: + res = 2 if 'sys.meta_path is None' in e.msg else 3 + assert not sys.modules + print(res) + + it = gen() + next(it) + """) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '1\n' + '2\n' * (INIT_LOOPS - 1)) + def test_static_types_inherited_slots(self): script = textwrap.dedent(""" import test.support From d7f80dd5f43407c720e45af83b403391a14284c8 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 30 Apr 2025 13:12:30 +0900 Subject: [PATCH 46/55] Restore and keep _Py_ID for now --- Include/internal/pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_runtime_init_generated.h | 1 + Include/internal/pycore_unicodeobject_generated.h | 4 ++++ Modules/_datetimemodule.c | 2 ++ 5 files changed, 9 insertions(+) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 53e7593e1584e4..e412db1de68f8b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -835,6 +835,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_call)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cafile)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index df8cae8c7efec3..2a6c2065af6bb9 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -326,6 +326,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(c_call) STRUCT_FOR_ID(c_exception) STRUCT_FOR_ID(c_return) + STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) STRUCT_FOR_ID(cafile) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index e083c858b4f5ea..2368157a4fd18b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -833,6 +833,7 @@ extern "C" { INIT_ID(c_call), \ INIT_ID(c_exception), \ INIT_ID(c_return), \ + INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ INIT_ID(cafile), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f5cf6ed5b4c6ac..72c3346328a552 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1092,6 +1092,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(cached_datetime_module); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_statements); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index cad082a1521ac1..ef19b1bca664b6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -125,6 +125,8 @@ get_module_state(PyObject *module) } +#define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) // unused + static PyObject * get_current_module(PyInterpreterState *interp, int *p_reloading) { From 23bdffd606c31eca8a2fa5c9d2b29c0f5ddf452c Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Wed, 30 Apr 2025 13:13:20 +0900 Subject: [PATCH 47/55] Add a demonstration to test_embed --- Lib/test/test_embed.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index afe2c2286b67ad..23199d50ac3d21 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -441,7 +441,7 @@ def test_datetime_reset_strptime(self): self.assertEqual(out, '20000101\n' * INIT_LOOPS) def test_datetime_capi_at_shutdown(self): - # gh-120782: Test the case where PyDateTime_IMPORT is called only once + # gh-120782: Current datetime test calls PyDateTime_IMPORT only once code = textwrap.dedent(""" import sys import _testcapi @@ -471,6 +471,34 @@ def gen(): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '1\n' + '2\n' * (INIT_LOOPS - 1)) + def test_datetime_capi_at_shutdown2(self): + # gh-120782: This PR allows PyDateTime_IMPORT to be called on restart + code = textwrap.dedent(""" + import sys + import _testcapi + _testcapi.test_datetime_capi() + assert '_datetime' in sys.modules + timedelta = _testcapi.get_capi_types()['timedelta'] + + def gen(): + try: + yield + finally: + assert not sys.modules + res = 1 + try: + timedelta(days=1) + except ImportError as e: + res = 2 if 'sys.meta_path is None' in e.msg else 3 + assert not sys.modules + print(res) + + it = gen() + next(it) + """) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(out, '1\n' * INIT_LOOPS) + def test_static_types_inherited_slots(self): script = textwrap.dedent(""" import test.support From ad7dfd2dfb012ba6c7a68f1996ab0057954f5cb0 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 1 May 2025 01:20:58 +0900 Subject: [PATCH 48/55] Introduce test_datetime_capi_newinterp() --- Lib/test/datetimetester.py | 11 ++++----- Lib/test/test_embed.py | 43 +++++------------------------------- Modules/_testcapi/datetime.c | 29 ++++++++++++++++++++++-- 3 files changed, 37 insertions(+), 46 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 9f21a23524b2da..d38168f567b3e1 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7178,13 +7178,13 @@ def assert_python_in_subinterp(self, check_if_ok, script, init='', _testcapi = importlib.util.module_from_spec(spec) spec.loader.exec_module(_testcapi) INDEX = $INDEX$ - setup = _testcapi.test_datetime_capi # call it if needed + setup = _testcapi.test_datetime_capi_newinterp # call it if needed $SCRIPT$ """ import _testcapi from test import support - setup = _testcapi.test_datetime_capi + setup = _testcapi.test_datetime_capi_newinterp $INIT$ for idx in range({repeat}): @@ -7335,16 +7335,13 @@ def test_static_type_on_subinterp(self): date = _testcapi.get_capi_types()['date'] date.today """) - with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - # FIXME: Segfault - self.assert_python_in_subinterp(False, script, 'setup()') - with_setup = 'setup()' + script with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): self.assert_python_in_subinterp(True, with_setup) with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): - # Check if PyDateTime_IMPORT is invoked not only once + # Fails if the setup() means test_datetime_capi() rather than + # test_datetime_capi_newinterp() self.assert_python_in_subinterp(True, with_setup, 'setup()') self.assert_python_in_subinterp(True, 'setup()', fini=with_setup) self.assert_python_in_subinterp(True, with_setup, repeat=2) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 23199d50ac3d21..e859fa78d11de8 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -441,15 +441,12 @@ def test_datetime_reset_strptime(self): self.assertEqual(out, '20000101\n' * INIT_LOOPS) def test_datetime_capi_at_shutdown(self): - # gh-120782: Current datetime test calls PyDateTime_IMPORT only once + # gh-132413: datetime module is currently tested in an interp's life. + # PyDateTime_IMPORT needs to be called at least once after the restart. code = textwrap.dedent(""" import sys import _testcapi - if not _testcapi.get_capi_types(): - _testcapi.test_datetime_capi() - assert '_datetime' in sys.modules - else: - assert '_datetime' not in sys.modules + _testcapi.test_datetime_capi_newinterp() timedelta = _testcapi.get_capi_types()['timedelta'] def gen(): @@ -457,40 +454,12 @@ def gen(): yield finally: assert not sys.modules - res = 1 + res = 0 try: timedelta(days=1) + res = 1 except ImportError as e: - res = 2 if 'sys.meta_path is None' in e.msg else 3 - assert not sys.modules - print(res) - - it = gen() - next(it) - """) - out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) - self.assertEqual(out, '1\n' + '2\n' * (INIT_LOOPS - 1)) - - def test_datetime_capi_at_shutdown2(self): - # gh-120782: This PR allows PyDateTime_IMPORT to be called on restart - code = textwrap.dedent(""" - import sys - import _testcapi - _testcapi.test_datetime_capi() - assert '_datetime' in sys.modules - timedelta = _testcapi.get_capi_types()['timedelta'] - - def gen(): - try: - yield - finally: - assert not sys.modules - res = 1 - try: - timedelta(days=1) - except ImportError as e: - res = 2 if 'sys.meta_path is None' in e.msg else 3 - assert not sys.modules + res = 2 print(res) it = gen() diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 42bce301a51277..375196e28fb727 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -11,6 +11,7 @@ test_datetime_capi(PyObject *self, PyObject *args) if (PyDateTimeAPI) { if (test_run_counter) { /* Probably regrtest.py -R */ + Py_RETURN_NONE; } else { PyErr_SetString(PyExc_AssertionError, @@ -19,7 +20,7 @@ test_datetime_capi(PyObject *self, PyObject *args) } } test_run_counter++; - PyDateTime_IMPORT; // Ensure interpreters individually import a module + PyDateTime_IMPORT; if (PyDateTimeAPI == NULL) { return NULL; @@ -34,6 +35,29 @@ test_datetime_capi(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +test_datetime_capi_newinterp(PyObject *self, PyObject *args) +{ + // Call PyDateTime_IMPORT at least once in each interpreter's life + if (PyDateTimeAPI != NULL && test_run_counter == 0) { + PyErr_SetString(PyExc_AssertionError, + "PyDateTime_CAPI somehow initialized"); + return NULL; + } + test_run_counter++; + PyDateTime_IMPORT; + + if (PyDateTimeAPI == NULL) { + return NULL; + } + assert(!PyType_HasFeature(PyDateTimeAPI->DateType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DateTimeType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->DeltaType, Py_TPFLAGS_HEAPTYPE)); + assert(!PyType_HasFeature(PyDateTimeAPI->TZInfoType, Py_TPFLAGS_HEAPTYPE)); + Py_RETURN_NONE; +} + /* Functions exposing the C API type checking for testing */ #define MAKE_DATETIME_CHECK_FUNC(check_method, exact_method) \ do { \ @@ -506,6 +530,7 @@ static PyMethodDef test_methods[] = { {"get_capi_types", get_capi_types, METH_NOARGS}, {"make_timezones_capi", make_timezones_capi, METH_NOARGS}, {"test_datetime_capi", test_datetime_capi, METH_NOARGS}, + {"test_datetime_capi_newinterp",test_datetime_capi_newinterp, METH_NOARGS}, {NULL}, }; @@ -526,7 +551,7 @@ _PyTestCapi_Init_DateTime(PyObject *mod) static int _testcapi_datetime_exec(PyObject *mod) { - // The execution does not invoke test_datetime_capi() + // The execution does not invoke PyDateTime_IMPORT return 0; } From 37b27747e7464a6a83377843fc07ee6ca63ace31 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 1 May 2025 09:22:48 +0900 Subject: [PATCH 49/55] Cleanup --- Lib/test/datetimetester.py | 43 +++++++++++++++++--------------------- Lib/test/test_embed.py | 2 +- Modules/_datetimemodule.c | 2 -- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index d38168f567b3e1..88c153f8379025 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7155,8 +7155,8 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) - def assert_python_in_subinterp(self, check_if_ok, script, init='', - fini='', repeat=1, config='isolated'): + def assert_python_ok_in_subinterp(self, script, init='', fini='', + repeat=1, config='isolated'): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. if sys.platform == "ios": @@ -7177,7 +7177,7 @@ def assert_python_in_subinterp(self, check_if_ok, script, init='', spec = importlib.util.spec_from_loader(fullname, loader) _testcapi = importlib.util.module_from_spec(spec) spec.loader.exec_module(_testcapi) - INDEX = $INDEX$ + run_counter = $RUN_COUNTER$ setup = _testcapi.test_datetime_capi_newinterp # call it if needed $SCRIPT$ """ @@ -7187,8 +7187,8 @@ def assert_python_in_subinterp(self, check_if_ok, script, init='', setup = _testcapi.test_datetime_capi_newinterp $INIT$ - for idx in range({repeat}): - subcode = subinterp_code.replace('$INDEX$', str(idx)) + for i in range(1, {1 + repeat}): + subcode = subinterp_code.replace('$RUN_COUNTER$', str(i)) if {_interpreters is None}: ret = support.run_in_subinterp(subcode) else: @@ -7200,12 +7200,7 @@ def assert_python_in_subinterp(self, check_if_ok, script, init='', ''') code = code.replace('$INIT$', init).replace('$FINI$', fini) code = code.replace('$SCRIPT$', script) - - if check_if_ok: - res = script_helper.assert_python_ok('-c', code) - else: - res = script_helper.assert_python_failure('-c', code) - return res + return script_helper.assert_python_ok('-c', code) def test_type_check_in_subinterp(self): script = textwrap.dedent(f""" @@ -7221,10 +7216,10 @@ def run(type_checker, obj): run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) """) - self.assert_python_in_subinterp(True, script) + self.assert_python_ok_in_subinterp(script) if _interpreters is not None: with self.subTest(name := 'legacy'): - self.assert_python_in_subinterp(True, script, config=name) + self.assert_python_ok_in_subinterp(script, config=name) class ExtensionModuleTests(unittest.TestCase): @@ -7233,8 +7228,8 @@ def setUp(self): if self.__class__.__name__.endswith('Pure'): self.skipTest('Not relevant in pure Python') - def assert_python_in_subinterp(self, *args, **kwargs): - return CapiTest.assert_python_in_subinterp(self, *args, **kwargs) + def assert_python_ok_in_subinterp(self, *args, **kwargs): + return CapiTest.assert_python_ok_in_subinterp(self, *args, **kwargs) @support.cpython_only def test_gh_120161(self): @@ -7337,25 +7332,25 @@ def test_static_type_on_subinterp(self): """) with_setup = 'setup()' + script with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - self.assert_python_in_subinterp(True, with_setup) + self.assert_python_ok_in_subinterp(with_setup) with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): # Fails if the setup() means test_datetime_capi() rather than # test_datetime_capi_newinterp() - self.assert_python_in_subinterp(True, with_setup, 'setup()') - self.assert_python_in_subinterp(True, 'setup()', fini=with_setup) - self.assert_python_in_subinterp(True, with_setup, repeat=2) + self.assert_python_ok_in_subinterp(with_setup, 'setup()') + self.assert_python_ok_in_subinterp('setup()', fini=with_setup) + self.assert_python_ok_in_subinterp(with_setup, repeat=2) with_import = 'import _datetime' + script with self.subTest('Explicit import'): - self.assert_python_in_subinterp(True, with_import, 'setup()') + self.assert_python_ok_in_subinterp(with_import, 'setup()') with_import = textwrap.dedent(""" timedelta = _testcapi.get_capi_types()['timedelta'] timedelta(days=1) """) + script with self.subTest('Implicit import'): - self.assert_python_in_subinterp(True, with_import, 'setup()') + self.assert_python_ok_in_subinterp(with_import, 'setup()') def test_static_type_at_shutdown(self): # gh-132413 @@ -7383,7 +7378,7 @@ def gen(): res = script_helper.assert_python_ok('-c', script) self.assertFalse(res.err) with self.subTest('Subinterpreter'): - res = self.assert_python_in_subinterp(True, script) + res = self.assert_python_ok_in_subinterp(script) self.assertFalse(res.err) script = textwrap.dedent(""" @@ -7404,12 +7399,12 @@ def gen(): next(it) """) with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - res = self.assert_python_in_subinterp(True, script, 'setup()') + res = self.assert_python_ok_in_subinterp(script, 'setup()') self.assertIn(b'ImportError: sys.meta_path is None', res.err) with_import = 'setup()' + script with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - res = self.assert_python_in_subinterp(True, with_import) + res = self.assert_python_ok_in_subinterp(with_import) self.assertFalse(res.err) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index e859fa78d11de8..d272e106d66b10 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -458,7 +458,7 @@ def gen(): try: timedelta(days=1) res = 1 - except ImportError as e: + except ImportError: res = 2 print(res) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index ef19b1bca664b6..08759fd03b39ba 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -12,7 +12,6 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyObject_Init() -#include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_time.h" // _PyTime_ObjectToTime_t() #include "pycore_unicodeobject.h" // _PyUnicode_Copy() @@ -156,7 +155,6 @@ _get_current_state(PyObject **p_mod) * so we must re-import the module. */ PyObject *mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { - assert(_Py_IsInterpreterFinalizing(interp)); return NULL; } st = get_module_state(mod); From f21f03f850844e53c7281e4db8188e1629662d1e Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Thu, 1 May 2025 21:46:14 +0900 Subject: [PATCH 50/55] Add a test to test_embed --- Lib/test/test_embed.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d272e106d66b10..6cfa9ee43b3e4b 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -440,6 +440,21 @@ def test_datetime_reset_strptime(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '20000101\n' * INIT_LOOPS) + def test_datetime_capi_type_address(self): + # Check if the C-API types keep their addresses until runtime shutdown + code = textwrap.dedent(""" + import _datetime as d + print( + f'{id(d.date)}' + f'{id(d.time)}' + f'{id(d.datetime)}' + f'{id(d.timedelta)}' + f'{id(d.tzinfo)}' + ) + """) + out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) + self.assertEqual(len(set(out.splitlines())), 1) + def test_datetime_capi_at_shutdown(self): # gh-132413: datetime module is currently tested in an interp's life. # PyDateTime_IMPORT needs to be called at least once after the restart. From 2b6c12a145f40a91678fcfd0137d7d896f9390e9 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 11 May 2025 03:41:00 +0900 Subject: [PATCH 51/55] Decref the module explicitly on exec error --- Modules/_datetimemodule.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 08759fd03b39ba..b2d4ff07766316 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7173,8 +7173,6 @@ create_timezone_from_delta(int days, int sec, int ms, int normalize) static int init_state(datetime_state *st, PyObject *module, PyObject *old_module) { - st->module = Py_NewRef(module); - /* Each module gets its own heap types. */ #define ADD_TYPE(FIELD, SPEC, BASE) \ do { \ @@ -7253,7 +7251,6 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) static int traverse_state(datetime_state *st, visitproc visit, void *arg) { - Py_VISIT(st->module); Py_VISIT(st->isocalendar_date_type); return 0; @@ -7262,7 +7259,6 @@ traverse_state(datetime_state *st, visitproc visit, void *arg) static int clear_state(datetime_state *st) { - Py_CLEAR(st->module); /* Invalidate first */ Py_CLEAR(st->isocalendar_date_type); Py_CLEAR(st->us_per_ms); Py_CLEAR(st->us_per_second); @@ -7339,6 +7335,7 @@ _datetime_exec(PyObject *module) } } + st->module = Py_NewRef(module); if (init_state(st, module, old_module) < 0) { goto error; } @@ -7453,6 +7450,7 @@ _datetime_exec(PyObject *module) goto finally; error: + Py_CLEAR(st->module); clear_state(st); finally: @@ -7471,6 +7469,7 @@ static int module_traverse(PyObject *mod, visitproc visit, void *arg) { datetime_state *st = get_module_state(mod); + Py_VISIT(st->module); traverse_state(st, visit, arg); return 0; } @@ -7479,6 +7478,7 @@ static int module_clear(PyObject *mod) { datetime_state *st = get_module_state(mod); + Py_CLEAR(st->module); clear_state(st); PyInterpreterState *interp = PyInterpreterState_Get(); From 8d94b2f3df600ce39ee5ee9aabd1fadc6ccac21f Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 11 May 2025 06:52:37 +0900 Subject: [PATCH 52/55] Focus on main interp's issue --- Lib/test/datetimetester.py | 132 +++++++---------------------------- Lib/test/test_embed.py | 8 +-- Modules/_datetimemodule.c | 1 + Modules/_testcapi/datetime.c | 28 +------- 4 files changed, 35 insertions(+), 134 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 88c153f8379025..2406a1f7ad13bb 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7155,8 +7155,7 @@ def test_datetime_from_timestamp(self): self.assertEqual(dt_orig, dt_rt) - def assert_python_ok_in_subinterp(self, script, init='', fini='', - repeat=1, config='isolated'): + def test_type_check_in_subinterp(self): # iOS requires the use of the custom framework loader, # not the ExtensionFileLoader. if sys.platform == "ios": @@ -7164,10 +7163,10 @@ def assert_python_ok_in_subinterp(self, script, init='', fini='', else: extension_loader = "ExtensionFileLoader" - code = textwrap.dedent(f''' - subinterp_code = """ + script = textwrap.dedent(f""" if {_interpreters is None}: - import _testcapi + import _testcapi as module + module.test_datetime_capi() else: import importlib.machinery import importlib.util @@ -7175,51 +7174,29 @@ def assert_python_ok_in_subinterp(self, script, init='', fini='', origin = importlib.util.find_spec('_testcapi').origin loader = importlib.machinery.{extension_loader}(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) - _testcapi = importlib.util.module_from_spec(spec) - spec.loader.exec_module(_testcapi) - run_counter = $RUN_COUNTER$ - setup = _testcapi.test_datetime_capi_newinterp # call it if needed - $SCRIPT$ - """ - - import _testcapi - from test import support - setup = _testcapi.test_datetime_capi_newinterp - $INIT$ + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) - for i in range(1, {1 + repeat}): - subcode = subinterp_code.replace('$RUN_COUNTER$', str(i)) - if {_interpreters is None}: - ret = support.run_in_subinterp(subcode) - else: - import _interpreters - config = _interpreters.new_config('{config}').__dict__ - ret = support.run_in_subinterp_with_config(subcode, **config) - assert ret == 0 - $FINI$ - ''') - code = code.replace('$INIT$', init).replace('$FINI$', fini) - code = code.replace('$SCRIPT$', script) - return script_helper.assert_python_ok('-c', code) - - def test_type_check_in_subinterp(self): - script = textwrap.dedent(f""" def run(type_checker, obj): if not type_checker(obj, True): raise TypeError(f'{{type(obj)}} is not C API type') - setup() import _datetime - run(_testcapi.datetime_check_date, _datetime.date.today()) - run(_testcapi.datetime_check_datetime, _datetime.datetime.now()) - run(_testcapi.datetime_check_time, _datetime.time(12, 30)) - run(_testcapi.datetime_check_delta, _datetime.timedelta(1)) - run(_testcapi.datetime_check_tzinfo, _datetime.tzinfo()) - """) - self.assert_python_ok_in_subinterp(script) - if _interpreters is not None: - with self.subTest(name := 'legacy'): - self.assert_python_ok_in_subinterp(script, config=name) + run(module.datetime_check_date, _datetime.date.today()) + run(module.datetime_check_datetime, _datetime.datetime.now()) + run(module.datetime_check_time, _datetime.time(12, 30)) + run(module.datetime_check_delta, _datetime.timedelta(1)) + run(module.datetime_check_tzinfo, _datetime.tzinfo()) + """) + if _interpreters is None: + ret = support.run_in_subinterp(script) + self.assertEqual(ret, 0) + else: + for name in ('isolated', 'legacy'): + with self.subTest(name): + config = _interpreters.new_config(name).__dict__ + ret = support.run_in_subinterp_with_config(script, **config) + self.assertEqual(ret, 0) class ExtensionModuleTests(unittest.TestCase): @@ -7228,9 +7205,6 @@ def setUp(self): if self.__class__.__name__.endswith('Pure'): self.skipTest('Not relevant in pure Python') - def assert_python_ok_in_subinterp(self, *args, **kwargs): - return CapiTest.assert_python_ok_in_subinterp(self, *args, **kwargs) - @support.cpython_only def test_gh_120161(self): with self.subTest('simple'): @@ -7325,33 +7299,6 @@ def test_no_leak(self): res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) self.assertIn(b'[0 refs, 0 blocks]', res.err) - def test_static_type_on_subinterp(self): - script = textwrap.dedent(""" - date = _testcapi.get_capi_types()['date'] - date.today - """) - with_setup = 'setup()' + script - with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - self.assert_python_ok_in_subinterp(with_setup) - - with self.subTest('[PyDateTime_IMPORT] main: yes, sub: yes'): - # Fails if the setup() means test_datetime_capi() rather than - # test_datetime_capi_newinterp() - self.assert_python_ok_in_subinterp(with_setup, 'setup()') - self.assert_python_ok_in_subinterp('setup()', fini=with_setup) - self.assert_python_ok_in_subinterp(with_setup, repeat=2) - - with_import = 'import _datetime' + script - with self.subTest('Explicit import'): - self.assert_python_ok_in_subinterp(with_import, 'setup()') - - with_import = textwrap.dedent(""" - timedelta = _testcapi.get_capi_types()['timedelta'] - timedelta(days=1) - """) + script - with self.subTest('Implicit import'): - self.assert_python_ok_in_subinterp(with_import, 'setup()') - def test_static_type_at_shutdown(self): # gh-132413 script = textwrap.dedent(""" @@ -7374,38 +7321,13 @@ def gen(): it = gen() next(it) """) - with self.subTest('MainInterpreter'): - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) - with self.subTest('Subinterpreter'): - res = self.assert_python_ok_in_subinterp(script) - self.assertFalse(res.err) - - script = textwrap.dedent(""" - import sys - timedelta = _testcapi.get_capi_types()['timedelta'] - - def gen(): - try: - yield - finally: - # Exceptions are ignored here - assert not sys.modules - td = timedelta(days=1) - assert td.days == 1 - assert not sys.modules + res = script_helper.assert_python_ok('-c', script) + self.assertFalse(res.err) - it = gen() - next(it) - """) - with self.subTest('[PyDateTime_IMPORT] main: yes, sub: no'): - res = self.assert_python_ok_in_subinterp(script, 'setup()') - self.assertIn(b'ImportError: sys.meta_path is None', res.err) - - with_import = 'setup()' + script - with self.subTest('[PyDateTime_IMPORT] main: no, sub: yes'): - res = self.assert_python_ok_in_subinterp(with_import) - self.assertFalse(res.err) + if support.Py_DEBUG: + with self.subTest('Refleak'): + res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) + self.assertIn(b'[0 refs, 0 blocks]', res.err) def load_tests(loader, standard_tests, pattern): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 28bd60ed9b5928..8908f0ede6616c 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -456,12 +456,12 @@ def test_datetime_capi_type_address(self): self.assertEqual(len(set(out.splitlines())), 1) def test_datetime_capi_at_shutdown(self): - # gh-132413: datetime module is currently tested in an interp's life. - # PyDateTime_IMPORT needs to be called at least once after the restart. + # gh-132413: Users need to call PyDateTime_IMPORT every time + # after starting an interpreter. code = textwrap.dedent(""" import sys import _testcapi - _testcapi.test_datetime_capi_newinterp() + _testcapi.test_datetime_capi() # PyDateTime_IMPORT only once timedelta = _testcapi.get_capi_types()['timedelta'] def gen(): @@ -481,7 +481,7 @@ def gen(): next(it) """) out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) - self.assertEqual(out, '1\n' * INIT_LOOPS) + self.assertEqual(out, '1\n' + '2\n' * (INIT_LOOPS - 1)) def test_static_types_inherited_slots(self): script = textwrap.dedent(""" diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index b2d4ff07766316..a603952af07c45 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7251,6 +7251,7 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) static int traverse_state(datetime_state *st, visitproc visit, void *arg) { + /* heap types */ Py_VISIT(st->isocalendar_date_type); return 0; diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 375196e28fb727..36fd9e50ca154c 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -35,29 +35,6 @@ test_datetime_capi(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject * -test_datetime_capi_newinterp(PyObject *self, PyObject *args) -{ - // Call PyDateTime_IMPORT at least once in each interpreter's life - if (PyDateTimeAPI != NULL && test_run_counter == 0) { - PyErr_SetString(PyExc_AssertionError, - "PyDateTime_CAPI somehow initialized"); - return NULL; - } - test_run_counter++; - PyDateTime_IMPORT; - - if (PyDateTimeAPI == NULL) { - return NULL; - } - assert(!PyType_HasFeature(PyDateTimeAPI->DateType, Py_TPFLAGS_HEAPTYPE)); - assert(!PyType_HasFeature(PyDateTimeAPI->TimeType, Py_TPFLAGS_HEAPTYPE)); - assert(!PyType_HasFeature(PyDateTimeAPI->DateTimeType, Py_TPFLAGS_HEAPTYPE)); - assert(!PyType_HasFeature(PyDateTimeAPI->DeltaType, Py_TPFLAGS_HEAPTYPE)); - assert(!PyType_HasFeature(PyDateTimeAPI->TZInfoType, Py_TPFLAGS_HEAPTYPE)); - Py_RETURN_NONE; -} - /* Functions exposing the C API type checking for testing */ #define MAKE_DATETIME_CHECK_FUNC(check_method, exact_method) \ do { \ @@ -530,7 +507,6 @@ static PyMethodDef test_methods[] = { {"get_capi_types", get_capi_types, METH_NOARGS}, {"make_timezones_capi", make_timezones_capi, METH_NOARGS}, {"test_datetime_capi", test_datetime_capi, METH_NOARGS}, - {"test_datetime_capi_newinterp",test_datetime_capi_newinterp, METH_NOARGS}, {NULL}, }; @@ -551,7 +527,9 @@ _PyTestCapi_Init_DateTime(PyObject *mod) static int _testcapi_datetime_exec(PyObject *mod) { - // The execution does not invoke PyDateTime_IMPORT + if (test_datetime_capi(NULL, NULL) == NULL) { + return -1; + } return 0; } From e1c9cdfbffd1bbb93a4dcd1559b5b63815ebcc9a Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sun, 11 May 2025 09:26:37 +0900 Subject: [PATCH 53/55] Tweak 2b6c12a --- Modules/_datetimemodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index a603952af07c45..9b01f44703d979 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7191,7 +7191,6 @@ init_state(datetime_state *st, PyObject *module, PyObject *old_module) assert(old_module != module); datetime_state *st_old = get_module_state(old_module); *st = (datetime_state){ - .module = st->module, .isocalendar_date_type = st->isocalendar_date_type, .us_per_ms = Py_NewRef(st_old->us_per_ms), .us_per_second = Py_NewRef(st_old->us_per_second), @@ -7336,7 +7335,6 @@ _datetime_exec(PyObject *module) } } - st->module = Py_NewRef(module); if (init_state(st, module, old_module) < 0) { goto error; } @@ -7446,12 +7444,12 @@ _datetime_exec(PyObject *module) if (set_current_module(interp, module) < 0) { goto error; } + st->module = Py_NewRef(module); rc = 0; goto finally; error: - Py_CLEAR(st->module); clear_state(st); finally: From 5ed5e69b8e82f339aebc997d77a2bc3fb0443633 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 13 May 2025 19:41:33 +0900 Subject: [PATCH 54/55] Smaller patch for tests --- Lib/test/datetimetester.py | 3 +-- Lib/test/test_embed.py | 2 +- Modules/_testcapi/datetime.c | 32 -------------------------------- 3 files changed, 2 insertions(+), 35 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 2406a1f7ad13bb..b2025131924bab 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7270,8 +7270,7 @@ def test_update_type_cache(self): assert isinstance(_datetime.timezone.utc, _datetime.tzinfo) del sys.modules['_datetime'] """) - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) + script_helper.assert_python_ok('-c', script) def test_module_free(self): script = textwrap.dedent(""" diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 8908f0ede6616c..cf0a7f2d88d5ee 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -462,7 +462,7 @@ def test_datetime_capi_at_shutdown(self): import sys import _testcapi _testcapi.test_datetime_capi() # PyDateTime_IMPORT only once - timedelta = _testcapi.get_capi_types()['timedelta'] + timedelta = type(_testcapi.get_delta_fromdsu(False, 1, 0, 0)) def gen(): try: diff --git a/Modules/_testcapi/datetime.c b/Modules/_testcapi/datetime.c index 36fd9e50ca154c..b800f9b8eb3473 100644 --- a/Modules/_testcapi/datetime.c +++ b/Modules/_testcapi/datetime.c @@ -453,37 +453,6 @@ test_PyDateTime_DELTA_GET(PyObject *self, PyObject *obj) return Py_BuildValue("(iii)", days, seconds, microseconds); } -static PyObject * -get_capi_types(PyObject *self, PyObject *args) -{ - if (PyDateTimeAPI == NULL) { - Py_RETURN_NONE; - } - PyObject *dict = PyDict_New(); - if (dict == NULL) { - return NULL; - } - if (PyDict_SetItemString(dict, "date", (PyObject *)PyDateTimeAPI->DateType) < 0) { - goto error; - } - if (PyDict_SetItemString(dict, "time", (PyObject *)PyDateTimeAPI->TimeType) < 0) { - goto error; - } - if (PyDict_SetItemString(dict, "datetime", (PyObject *)PyDateTimeAPI->DateTimeType) < 0) { - goto error; - } - if (PyDict_SetItemString(dict, "timedelta", (PyObject *)PyDateTimeAPI->DeltaType) < 0) { - goto error; - } - if (PyDict_SetItemString(dict, "tzinfo", (PyObject *)PyDateTimeAPI->TZInfoType) < 0) { - goto error; - } - return dict; -error: - Py_DECREF(dict); - return NULL; -} - static PyMethodDef test_methods[] = { {"PyDateTime_DATE_GET", test_PyDateTime_DATE_GET, METH_O}, {"PyDateTime_DELTA_GET", test_PyDateTime_DELTA_GET, METH_O}, @@ -504,7 +473,6 @@ static PyMethodDef test_methods[] = { {"get_time_fromtimeandfold", get_time_fromtimeandfold, METH_VARARGS}, {"get_timezone_utc_capi", get_timezone_utc_capi, METH_VARARGS}, {"get_timezones_offset_zero", get_timezones_offset_zero, METH_NOARGS}, - {"get_capi_types", get_capi_types, METH_NOARGS}, {"make_timezones_capi", make_timezones_capi, METH_NOARGS}, {"test_datetime_capi", test_datetime_capi, METH_NOARGS}, {NULL}, From f22edc1e32721d5dcd6c6b297991fcb567b8acce Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Tue, 24 Jun 2025 05:01:30 +0900 Subject: [PATCH 55/55] minimize test --- Lib/test/datetimetester.py | 45 ++++---------------------------------- Lib/test/test_embed.py | 43 ------------------------------------ 2 files changed, 4 insertions(+), 84 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 3b96cd856d853e..49e78f6f9a74cd 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7275,36 +7275,9 @@ def test_update_type_cache(self): """) script_helper.assert_python_ok('-c', script) - def test_module_free(self): - script = textwrap.dedent(""" - import sys - import gc - import weakref - ws = weakref.WeakSet() - for _ in range(3): - import _datetime - timedelta = _datetime.timedelta # static type - ws.add(_datetime) - del sys.modules['_datetime'] - del _datetime - gc.collect() - assert len(ws) == 0 - """) - script_helper.assert_python_ok('-c', script) - - @unittest.skipIf(not support.Py_DEBUG, "Debug builds only") - def test_no_leak(self): - script = textwrap.dedent(""" - import datetime - datetime.datetime.strptime('20000101', '%Y%m%d').strftime('%Y%m%d') - """) - res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) - self.assertIn(b'[0 refs, 0 blocks]', res.err) - def test_static_type_at_shutdown(self): # gh-132413 script = textwrap.dedent(""" - import sys import _datetime timedelta = _datetime.timedelta @@ -7312,24 +7285,14 @@ def gen(): try: yield finally: - # Exceptions are ignored here - assert not sys.modules - td = _datetime.timedelta(days=1) - assert td.days == 1 - td = timedelta(days=1) - assert td.days == 1 - assert not sys.modules + # sys.modules is empty + _datetime.timedelta(days=1) + timedelta(days=1) it = gen() next(it) """) - res = script_helper.assert_python_ok('-c', script) - self.assertFalse(res.err) - - if support.Py_DEBUG: - with self.subTest('Refleak'): - res = script_helper.assert_python_ok('-X', 'showrefcount', '-c', script) - self.assertIn(b'[0 refs, 0 blocks]', res.err) + script_helper.assert_python_ok('-c', script) def load_tests(loader, standard_tests, pattern): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 8717541de57dce..46222e521aead8 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -440,49 +440,6 @@ def test_datetime_reset_strptime(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '20000101\n' * INIT_LOOPS) - def test_datetime_capi_type_address(self): - # Check if the C-API types keep their addresses until runtime shutdown - code = textwrap.dedent(""" - import _datetime as d - print( - f'{id(d.date)}' - f'{id(d.time)}' - f'{id(d.datetime)}' - f'{id(d.timedelta)}' - f'{id(d.tzinfo)}' - ) - """) - out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) - self.assertEqual(len(set(out.splitlines())), 1) - - def test_datetime_capi_at_shutdown(self): - # gh-132413: Users need to call PyDateTime_IMPORT every time - # after starting an interpreter. - code = textwrap.dedent(""" - import sys - import _testcapi - _testcapi.test_datetime_capi() # PyDateTime_IMPORT only once - timedelta = type(_testcapi.get_delta_fromdsu(False, 1, 0, 0)) - - def gen(): - try: - yield - finally: - assert not sys.modules - res = 0 - try: - timedelta(days=1) - res = 1 - except ImportError: - res = 2 - print(res) - - it = gen() - next(it) - """) - out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) - self.assertEqual(out, '1\n' + '2\n' * (INIT_LOOPS - 1)) - def test_static_types_inherited_slots(self): script = textwrap.dedent(""" import test.support pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy