diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 02a7b5d45b4627..57022de2b0e5ca 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,17 +4,23 @@ # It uses the same pattern rule for gitignore file # https://git-scm.com/docs/gitignore#_pattern_format -# GitHub +# Azure Pipelines +.azure-pipelines/ @AA-Turner + +# GitHub & related scripts .github/** @ezio-melotti @hugovk @AA-Turner +Tools/build/compute-changes.py @AA-Turner +Tools/build/verify_ensurepip_wheels.py @AA-Turner # pre-commit -.pre-commit-config.yaml @hugovk @AlexWaygood +.pre-commit-config.yaml @hugovk .ruff.toml @hugovk @AlexWaygood @AA-Turner -# Build system -configure* @erlend-aasland @corona10 -Makefile.pre.in @erlend-aasland -Modules/Setup* @erlend-aasland +# Build system (autotools) +configure* @erlend-aasland @corona10 @AA-Turner +Makefile.pre.in @erlend-aasland @AA-Turner +Modules/Setup* @erlend-aasland @AA-Turner +Tools/build/regen-configure.sh @AA-Turner # argparse **/*argparse* @savannahostrowski @@ -67,6 +73,7 @@ Doc/make.bat @AA-Turner @hugovk Doc/requirements.txt @AA-Turner @hugovk Doc/_static/** @AA-Turner @hugovk Doc/tools/** @AA-Turner @hugovk +.readthedocs.yml @AA-Turner # runtime state/lifecycle **/*pylifecycle* @ericsnowcurrently @ZeroIntensity @@ -155,6 +162,10 @@ Doc/c-api/module.rst @ericsnowcurrently **/*importlib/resources/* @jaraco @warsaw @FFY00 **/*importlib/metadata/* @jaraco @warsaw +# Calendar +Lib/calendar.py @AA-Turner +Lib/test/test_calendar.py @AA-Turner + # Dates and times **/*datetime* @pganssle @abalkin **/*str*time* @pganssle @abalkin @@ -205,6 +216,11 @@ Lib/test/test_ast/ @eclips4 @tomasr8 # multiprocessing **/*multiprocessing* @gpshead +# pydoc +Lib/pydoc.py @AA-Turner +Lib/pydoc_data/ @AA-Turner +Lib/test/test_pydoc/ @AA-Turner + # SQLite 3 **/*sqlite* @berkerpeksag @erlend-aasland @@ -217,6 +233,11 @@ Lib/test/test_ast/ @eclips4 @tomasr8 **/*pdb* @gaogaotiantian **/*bdb* @gaogaotiantian +# types +Lib/test/test_types.py @AA-Turner +Lib/types.py @AA-Turner +Modules/_typesmodule.c @AA-Turner + # Limited C API & stable ABI Tools/build/stable_abi.py @encukou Misc/stable_abi.toml @encukou @@ -234,6 +255,11 @@ Doc/c-api/stable.rst @encukou /Tools/msi/ @python/windows-team /Tools/nuget/ @python/windows-team +# Zstandard +Lib/compression/zstd/ @AA-Turner +Lib/test/test_zstd.py @AA-Turner +Modules/_zstd/ @AA-Turner + # Misc **/*itertools* @rhettinger **/*collections* @rhettinger @@ -266,6 +292,9 @@ Doc/c-api/stable.rst @encukou **/*cjkcodecs* @corona10 +# Patchcheck +Tools/patchcheck/ @AA-Turner + # macOS /Mac/ @python/macos-team **/*osx_support* @python/macos-team @@ -277,9 +306,9 @@ Doc/c-api/stable.rst @encukou **/*zipfile/_path/* @jaraco # Argument Clinic -/Tools/clinic/** @erlend-aasland -/Lib/test/test_clinic.py @erlend-aasland -Doc/howto/clinic.rst @erlend-aasland +/Tools/clinic/** @erlend-aasland @AA-Turner +/Lib/test/test_clinic.py @erlend-aasland @AA-Turner +Doc/howto/clinic.rst @erlend-aasland @AA-Turner # Subinterpreters **/*interpreteridobject.* @ericsnowcurrently @@ -323,6 +352,7 @@ Lib/test/test_configparser.py @jaraco # Doc sections Doc/reference/ @willingc @AA-Turner +Doc/whatsnew/ @AA-Turner **/*weakref* @kumaraditya303 @@ -336,7 +366,7 @@ Modules/_xxtestfuzz/ @ammaraskar # t-strings **/*interpolationobject* @lysnikolaou **/*templateobject* @lysnikolaou -**/*templatelib* @lysnikolaou +**/*templatelib* @lysnikolaou @AA-Turner **/*tstring* @lysnikolaou # Remote debugging @@ -346,3 +376,6 @@ Modules/_remote_debugging_module.c @pablogsal @ambv @1st1 # gettext **/*gettext* @tomasr8 + +# Internal Docs +InternalDocs/ @AA-Turner diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 2ef9cdc191481e..5b86302bdd1ec4 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -4,7 +4,7 @@ Contributing to Python Build Status ------------ -- `Buildbot status overview `_ +- `Buildbot status overview `_ - `GitHub Actions status `_ diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 947badff816c09..51a069d857f2a3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -117,6 +117,10 @@ jobs: find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete brew install llvm@${{ matrix.llvm }} export SDKROOT="$(xcrun --show-sdk-path)" + # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to + # make sure we don't break downstream distributors (like uv): + export CFLAGS_JIT='-Werror=unguarded-availability' + export MACOSX_DEPLOYMENT_TARGET=10.15 ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 7b9dc4818577eb..65154aae4c41d5 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -75,18 +75,6 @@ jobs: --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit - - name: 'Build EPUB documentation' - continue-on-error: true - run: | - set -Eeuo pipefail - make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet" epub - pip install epubcheck - epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt - - name: 'Check for fatal errors in EPUB' - if: github.event_name == 'pull_request' - continue-on-error: true # until gh-136155 is fixed - run: | - python Doc/tools/check-epub.py # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: @@ -114,3 +102,30 @@ jobs: # Use "xvfb-run" since some doctest tests open GUI windows - name: 'Run documentation doctest' run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="--fail-on-warning" doctest + + check-epub: + name: 'Check EPUB' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: 'Set up Python' + uses: actions/setup-python@v5 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: | + make -C Doc/ venv + python -m pip install epubcheck + - name: 'Build EPUB documentation' + run: make -C Doc/ PYTHON=../python epub + - name: 'Run epubcheck' + continue-on-error: true + run: epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt + - run: cat Doc/epubcheck.txt + - name: 'Check for fatal errors in EPUB' + run: python Doc/tools/check-epub.py diff --git a/.well-known/funding-manifest-urls b/.well-known/funding-manifest-urls new file mode 100644 index 00000000000000..d56ca9eb52f004 --- /dev/null +++ b/.well-known/funding-manifest-urls @@ -0,0 +1 @@ +https://www.python.org/funding.json diff --git a/Android/android.py b/Android/android.py index 75f73cd30993da..e6090aa1d80db0 100755 --- a/Android/android.py +++ b/Android/android.py @@ -187,7 +187,7 @@ def unpack_deps(host, prefix_dir): os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4", - "sqlite-3.49.1-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: + "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") shutil.unpack_archive(filename) diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 16bd79475dc1e6..a1fd27ad0acd2e 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -43,24 +43,36 @@ pointers. This is consistent throughout the API. Return the sum of two complex numbers, using the C :c:type:`Py_complex` representation. + .. deprecated:: 3.15 + This function is :term:`soft deprecated`. + .. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) Return the difference between two complex numbers, using the C :c:type:`Py_complex` representation. + .. deprecated:: 3.15 + This function is :term:`soft deprecated`. + .. c:function:: Py_complex _Py_c_neg(Py_complex num) Return the negation of the complex number *num*, using the C :c:type:`Py_complex` representation. + .. deprecated:: 3.15 + This function is :term:`soft deprecated`. + .. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) Return the product of two complex numbers, using the C :c:type:`Py_complex` representation. + .. deprecated:: 3.15 + This function is :term:`soft deprecated`. + .. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) @@ -70,6 +82,9 @@ pointers. This is consistent throughout the API. If *divisor* is null, this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. + .. deprecated:: 3.15 + This function is :term:`soft deprecated`. + .. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) @@ -81,6 +96,19 @@ pointers. This is consistent throughout the API. Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + .. deprecated:: 3.15 + This function is :term:`soft deprecated`. + + +.. c:function:: double _Py_c_abs(Py_complex num) + + Return the absolute value of the complex number *num*. + + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + + .. deprecated:: 3.15 + This function is :term:`soft deprecated`. + Complex Numbers as Python Objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -90,6 +118,16 @@ Complex Numbers as Python Objects This subtype of :c:type:`PyObject` represents a Python complex number object. + .. c:member:: Py_complex cval + + The complex number value, using the C :c:type:`Py_complex` representation. + + .. deprecated-removed:: next 3.20 + Use :c:func:`PyComplex_AsCComplex` and + :c:func:`PyComplex_FromCComplex` to convert a + Python complex number to/from the C :c:type:`Py_complex` + representation. + .. c:var:: PyTypeObject PyComplex_Type diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 2d0bda76697e81..31e2cd57034467 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -372,6 +372,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Set *\*value* to a signed C :c:expr:`int32_t` or :c:expr:`int64_t` representation of *obj*. + If *obj* is not an instance of :c:type:`PyLongObject`, first call its + :meth:`~object.__index__` method (if present) to convert it to a + :c:type:`PyLongObject`. + If the *obj* value is out of range, raise an :exc:`OverflowError`. Set *\*value* and return ``0`` on success. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 55f0d0f9fb7ff8..78599e704b1317 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -197,7 +197,7 @@ Object Protocol in favour of using :c:func:`PyObject_DelAttr`, but there are currently no plans to remove it. - The function must not be called with ``NULL`` *v* and an an exception set. + The function must not be called with a ``NULL`` *v* and an exception set. This case can arise from forgetting ``NULL`` checks and would delete the attribute. @@ -214,7 +214,7 @@ Object Protocol If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. - The function must not be called with ``NULL`` *v* and an an exception set. + The function must not be called with a ``NULL`` *v* and an exception set. This case can arise from forgetting ``NULL`` checks and would delete the attribute. diff --git a/Doc/conf.py b/Doc/conf.py index 1c1f36e5bc0737..35e0b3eaeafe94 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -567,14 +567,11 @@ 'image': '_static/og-image.png', 'line_color': '#3776ab', } -if 'builder_html' in tags: # noqa: F821 - ogp_custom_meta_tags = [ - '', - ] - if 'create-social-cards' not in tags: # noqa: F821 - # Define a static preview image when not creating social cards - ogp_image = '_static/og-image.png' - ogp_custom_meta_tags += [ - '', - '', - ] +ogp_custom_meta_tags = ('',) +if 'create-social-cards' not in tags: # noqa: F821 + # Define a static preview image when not creating social cards + ogp_image = '_static/og-image.png' + ogp_custom_meta_tags += ( + '', + '', + ) diff --git a/Doc/deprecations/c-api-pending-removal-in-3.20.rst b/Doc/deprecations/c-api-pending-removal-in-3.20.rst new file mode 100644 index 00000000000000..82f975d6ed4020 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.20.rst @@ -0,0 +1,7 @@ +Pending removal in Python 3.20 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The ``cval`` field in :c:type:`PyComplexObject` (:gh:`128813`). + Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` + to convert a Python complex number to/from the C :c:type:`Py_complex` + representation. diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index d064f2bec42c22..c6e05c176b2aa1 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -18,4 +18,6 @@ C API deprecations .. include:: c-api-pending-removal-in-3.18.rst +.. include:: c-api-pending-removal-in-3.20.rst + .. include:: c-api-pending-removal-in-future.rst diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index a89a69043c0f9f..17c6fb224265ca 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -75,12 +75,37 @@ the module and a copyright notice if you like). See :ref:`arg-parsing-string-and-buffers` for a description of this macro. All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or -``PY``, except those defined in standard header files. For convenience, and -since they are used extensively by the Python interpreter, ``"Python.h"`` -includes a few standard header files: ````, ````, -````, and ````. If the latter header file does not exist on -your system, it declares the functions :c:func:`malloc`, :c:func:`free` and -:c:func:`realloc` directly. +``PY``, except those defined in standard header files. + +.. tip:: + + For backward compatibility, :file:`Python.h` includes several standard header files. + C extensions should include the standard headers that they use, + and should not rely on these implicit includes. + If using the limited C API version 3.13 or newer, the implicit includes are: + + * ```` + * ```` (on Windows) + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` (if present) + + If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.12 or older, + the headers below are also included: + + * ```` + * ```` (on POSIX) + + If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.10 or older, + the headers below are also included: + + * ```` + * ```` + * ```` + * ```` The next thing we add to our module file is the C function that will be called when the Python expression ``spam.system(string)`` is evaluated (we'll see diff --git a/Doc/glossary.rst b/Doc/glossary.rst index b7bd547d38fd1e..69e8857af894e4 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -435,6 +435,11 @@ Glossary with :term:`abstract base classes `.) Instead, it typically employs :func:`hasattr` tests or :term:`EAFP` programming. + dunder + An informal short-hand for "double underscore", used when talking about a + :term:`special method`. For example, ``__init__`` is often pronounced + "dunder init". + EAFP Easier to ask for forgiveness than permission. This common Python coding style assumes the existence of valid keys or attributes and catches @@ -462,6 +467,7 @@ Glossary core and with user code. f-string + f-strings String literals prefixed with ``f`` or ``F`` are commonly called "f-strings" which is short for :ref:`formatted string literals `. See also :pep:`498`. @@ -1323,6 +1329,7 @@ Glossary See also :term:`borrowed reference`. t-string + t-strings String literals prefixed with ``t`` or ``T`` are commonly called "t-strings" which is short for :ref:`template string literals `. @@ -1472,6 +1479,11 @@ Glossary A computer defined entirely in software. Python's virtual machine executes the :term:`bytecode` emitted by the bytecode compiler. + walrus operator + A light-hearted way to refer to the :ref:`assignment expression + ` operator ``:=`` because it looks a bit like a + walrus if you turn your head. + Zen of Python Listing of Python design principles and philosophies that are helpful in understanding and using the language. The listing can be found by typing diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst new file mode 100644 index 00000000000000..d68f7cc6921fc9 --- /dev/null +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -0,0 +1,606 @@ +.. _a-conceptual-overview-of-asyncio: + +**************************************** +A Conceptual Overview of :mod:`!asyncio` +**************************************** + +This :ref:`HOWTO ` article seeks to help you build a sturdy mental +model of how :mod:`asyncio` fundamentally works, helping you understand the +how and why behind the recommended patterns. + +You might be curious about some key :mod:`!asyncio` concepts. +You'll be comfortably able to answer these questions by the end of this +article: + +- What's happening behind the scenes when an object is awaited? +- How does :mod:`!asyncio` differentiate between a task which doesn't need + CPU-time (such as a network request or file read) as opposed to a task that + does (such as computing n-factorial)? +- How to write an asynchronous variant of an operation, such as + an async sleep or database request. + +.. seealso:: + + * The `guide `_ that inspired this HOWTO article, by Alexander Nordin. + * This in-depth `YouTube tutorial series `_ on + ``asyncio`` created by Python core team member, Łukasz Langa. + * `500 Lines or Less: A Web Crawler With asyncio Coroutines `_ by A. + Jesse Jiryu Davis and Guido van Rossum. + +-------------------------------------------- +A conceptual overview part 1: the high-level +-------------------------------------------- + +In part 1, we'll cover the main, high-level building blocks of :mod:`!asyncio`: +the event loop, coroutine functions, coroutine objects, tasks and ``await``. + +========== +Event Loop +========== + +Everything in :mod:`!asyncio` happens relative to the event loop. +It's the star of the show. +It's like an orchestra conductor. +It's behind the scenes managing resources. +Some power is explicitly granted to it, but a lot of its ability to get things +done comes from the respect and cooperation of its worker bees. + +In more technical terms, the event loop contains a collection of jobs to be run. +Some jobs are added directly by you, and some indirectly by :mod:`!asyncio`. +The event loop takes a job from its backlog of work and invokes it (or "gives +it control"), similar to calling a function, and then that job runs. +Once it pauses or completes, it returns control to the event loop. +The event loop will then select another job from its pool and invoke it. +You can *roughly* think of the collection of jobs as a queue: jobs are added and +then processed one at a time, generally (but not always) in order. +This process repeats indefinitely with the event loop cycling endlessly +onwards. +If there are no more jobs pending execution, the event loop is smart enough to +rest and avoid needlessly wasting CPU cycles, and will come back when there's +more work to be done. + +Effective execution relies on jobs sharing well and cooperating; a greedy job +could hog control and leave the other jobs to starve, rendering the overall +event loop approach rather useless. + +:: + + import asyncio + + # This creates an event loop and indefinitely cycles through + # its collection of jobs. + event_loop = asyncio.new_event_loop() + event_loop.run_forever() + +===================================== +Asynchronous functions and coroutines +===================================== + +This is a basic, boring Python function:: + + def hello_printer(): + print( + "Hi, I am a lowly, simple printer, though I have all I " + "need in life -- \nfresh paper and my dearly beloved octopus " + "partner in crime." + ) + +Calling a regular function invokes its logic or body:: + + >>> hello_printer() + Hi, I am a lowly, simple printer, though I have all I need in life -- + fresh paper and my dearly beloved octopus partner in crime. + +The :ref:`async def `, as opposed to just a plain ``def``, makes +this an asynchronous function (or "coroutine function"). +Calling it creates and returns a :ref:`coroutine ` object. + +:: + + async def loudmouth_penguin(magic_number: int): + print( + "I am a super special talking penguin. Far cooler than that printer. " + f"By the way, my lucky number is: {magic_number}." + ) + +Calling the async function, ``loudmouth_penguin``, does not execute the print statement; +instead, it creates a coroutine object:: + + >>> loudmouth_penguin(magic_number=3) + + +The terms "coroutine function" and "coroutine object" are often conflated +as coroutine. +That can be confusing! +In this article, coroutine specifically refers to a coroutine object, or more +precisely, an instance of :data:`types.CoroutineType` (native coroutine). +Note that coroutines can also exist as instances of +:class:`collections.abc.Coroutine` -- a distinction that matters for type +checking. + +A coroutine represents the function's body or logic. +A coroutine has to be explicitly started; again, merely creating the coroutine +does not start it. +Notably, the coroutine can be paused and resumed at various points within the +function's body. +That pausing and resuming ability is what allows for asynchronous behavior! + +Coroutines and coroutine functions were built by leveraging the functionality +of :term:`generators ` and +:term:`generator functions `. +Recall, a generator function is a function that :keyword:`yield`\s, like this +one:: + + def get_random_number(): + # This would be a bad random number generator! + print("Hi") + yield 1 + print("Hello") + yield 7 + print("Howdy") + yield 4 + ... + +Similar to a coroutine function, calling a generator function does not run it. +Instead, it creates a generator object:: + + >>> get_random_number() + + +You can proceed to the next ``yield`` of a generator by using the +built-in function :func:`next`. +In other words, the generator runs, then pauses. +For example:: + + >>> generator = get_random_number() + >>> next(generator) + Hi + 1 + >>> next(generator) + Hello + 7 + +===== +Tasks +===== + +Roughly speaking, :ref:`tasks ` are coroutines (not coroutine +functions) tied to an event loop. +A task also maintains a list of callback functions whose importance will become +clear in a moment when we discuss :keyword:`await`. +The recommended way to create tasks is via :func:`asyncio.create_task`. + +Creating a task automatically schedules it for execution (by adding a +callback to run it in the event loop's to-do list, that is, collection of jobs). + +Since there's only one event loop (in each thread), :mod:`!asyncio` takes care of +associating the task with the event loop for you. As such, there's no need +to specify the event loop. + +:: + + coroutine = loudmouth_penguin(magic_number=5) + # This creates a Task object and schedules its execution via the event loop. + task = asyncio.create_task(coroutine) + +Earlier, we manually created the event loop and set it to run forever. +In practice, it's recommended to use (and common to see) :func:`asyncio.run`, +which takes care of managing the event loop and ensuring the provided +coroutine finishes before advancing. +For example, many async programs follow this setup:: + + import asyncio + + async def main(): + # Perform all sorts of wacky, wild asynchronous things... + ... + + if __name__ == "__main__": + asyncio.run(main()) + # The program will not reach the following print statement until the + # coroutine main() finishes. + print("coroutine main() is done!") + +It's important to be aware that the task itself is not added to the event loop, +only a callback to the task is. +This matters if the task object you created is garbage collected before it's +called by the event loop. +For example, consider this program: + +.. code-block:: + :linenos: + + async def hello(): + print("hello!") + + async def main(): + asyncio.create_task(hello()) + # Other asynchronous instructions which run for a while + # and cede control to the event loop... + ... + + asyncio.run(main()) + +Because there's no reference to the task object created on line 5, it *might* +be garbage collected before the event loop invokes it. +Later instructions in the coroutine ``main()`` hand control back to the event +loop so it can invoke other jobs. +When the event loop eventually tries to run the task, it might fail and +discover the task object does not exist! +This can also happen even if a coroutine keeps a reference to a task but +completes before that task finishes. +When the coroutine exits, local variables go out of scope and may be subject +to garbage collection. +In practice, ``asyncio`` and Python's garbage collector work pretty hard to +ensure this sort of thing doesn't happen. +But that's no reason to be reckless! + +===== +await +===== + +:keyword:`await` is a Python keyword that's commonly used in one of two +different ways:: + + await task + await coroutine + +In a crucial way, the behavior of ``await`` depends on the type of object +being awaited. + +Awaiting a task will cede control from the current task or coroutine to +the event loop. +In the process of relinquishing control, a few important things happen. +We'll use the following code example to illustrate:: + + async def plant_a_tree(): + dig_the_hole_task = asyncio.create_task(dig_the_hole()) + await dig_the_hole_task + + # Other instructions associated with planting a tree. + ... + +In this example, imagine the event loop has passed control to the start of the +coroutine ``plant_a_tree()``. +As seen above, the coroutine creates a task and then awaits it. +The ``await dig_the_hole_task`` instruction adds a callback (which will resume +``plant_a_tree()``) to the ``dig_the_hole_task`` object's list of callbacks. +And then, the instruction cedes control to the event loop. +Some time later, the event loop will pass control to ``dig_the_hole_task`` +and the task will finish whatever it needs to do. +Once the task finishes, it will add its various callbacks to the event loop, +in this case, a call to resume ``plant_a_tree()``. + +Generally speaking, when the awaited task finishes (``dig_the_hole_task``), +the original task or coroutine (``plant_a_tree()``) is added back to the event +loops to-do list to be resumed. + +This is a basic, yet reliable mental model. +In practice, the control handoffs are slightly more complex, but not by much. +In part 2, we'll walk through the details that make this possible. + +**Unlike tasks, awaiting a coroutine does not hand control back to the event +loop!** +Wrapping a coroutine in a task first, then awaiting that would cede +control. +The behavior of ``await coroutine`` is effectively the same as invoking a +regular, synchronous Python function. +Consider this program:: + + import asyncio + + async def coro_a(): + print("I am coro_a(). Hi!") + + async def coro_b(): + print("I am coro_b(). I sure hope no one hogs the event loop...") + + async def main(): + task_b = asyncio.create_task(coro_b()) + num_repeats = 3 + for _ in range(num_repeats): + await coro_a() + await task_b + + asyncio.run(main()) + +The first statement in the coroutine ``main()`` creates ``task_b`` and schedules +it for execution via the event loop. +Then, ``coro_a()`` is repeatedly awaited. Control never cedes to the +event loop which is why we see the output of all three ``coro_a()`` +invocations before ``coro_b()``'s output: + +.. code-block:: none + + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_b(). I sure hope no one hogs the event loop... + +If we change ``await coro_a()`` to ``await asyncio.create_task(coro_a())``, the +behavior changes. +The coroutine ``main()`` cedes control to the event loop with that statement. +The event loop then proceeds through its backlog of work, calling ``task_b`` +and then the task which wraps ``coro_a()`` before resuming the coroutine +``main()``. + +.. code-block:: none + + I am coro_b(). I sure hope no one hogs the event loop... + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + +This behavior of ``await coroutine`` can trip a lot of people up! +That example highlights how using only ``await coroutine`` could +unintentionally hog control from other tasks and effectively stall the event +loop. +:func:`asyncio.run` can help you detect such occurences via the +``debug=True`` flag which accordingly enables +:ref:`debug mode `. +Among other things, it will log any coroutines that monopolize execution for +100ms or longer. + +The design intentionally trades off some conceptual clarity around usage of +``await`` for improved performance. +Each time a task is awaited, control needs to be passed all the way up the +call stack to the event loop. +That might sound minor, but in a large program with many ``await``'s and a deep +callstack that overhead can add up to a meaningful performance drag. + +------------------------------------------------ +A conceptual overview part 2: the nuts and bolts +------------------------------------------------ + +Part 2 goes into detail on the mechanisms :mod:`!asyncio` uses to manage +control flow. +This is where the magic happens. +You'll come away from this section knowing what ``await`` does behind the scenes +and how to make your own asynchronous operators. + +================================ +The inner workings of coroutines +================================ + +:mod:`!asyncio` leverages four components to pass around control. + +:meth:`coroutine.send(arg) ` is the method used to start or +resume a coroutine. +If the coroutine was paused and is now being resumed, the argument ``arg`` +will be sent in as the return value of the ``yield`` statement which originally +paused it. +If the coroutine is being used for the first time (as opposed to being resumed) +``arg`` must be ``None``. + +.. code-block:: + :linenos: + + class Rock: + def __await__(self): + value_sent_in = yield 7 + print(f"Rock.__await__ resuming with value: {value_sent_in}.") + return value_sent_in + + async def main(): + print("Beginning coroutine main().") + rock = Rock() + print("Awaiting rock...") + value_from_rock = await rock + print(f"Coroutine received value: {value_from_rock} from rock.") + return 23 + + coroutine = main() + intermediate_result = coroutine.send(None) + print(f"Coroutine paused and returned intermediate value: {intermediate_result}.") + + print(f"Resuming coroutine and sending in value: 42.") + try: + coroutine.send(42) + except StopIteration as e: + returned_value = e.value + print(f"Coroutine main() finished and provided value: {returned_value}.") + +:ref:`yield `, like usual, pauses execution and returns control +to the caller. +In the example above, the ``yield``, on line 3, is called by +``... = await rock`` on line 11. +More broadly speaking, ``await`` calls the :meth:`~object.__await__` method of +the given object. +``await`` also does one more very special thing: it propagates (or "passes +along") any ``yield``\ s it receives up the call-chain. +In this case, that's back to ``... = coroutine.send(None)`` on line 16. + +The coroutine is resumed via the ``coroutine.send(42)`` call on line 21. +The coroutine picks back up from where it ``yield``\ ed (or paused) on line 3 +and executes the remaining statements in its body. +When a coroutine finishes, it raises a :exc:`StopIteration` exception with the +return value attached in the :attr:`~StopIteration.value` attribute. + +That snippet produces this output: + +.. code-block:: none + + Beginning coroutine main(). + Awaiting rock... + Coroutine paused and returned intermediate value: 7. + Resuming coroutine and sending in value: 42. + Rock.__await__ resuming with value: 42. + Coroutine received value: 42 from rock. + Coroutine main() finished and provided value: 23. + +It's worth pausing for a moment here and making sure you followed the various +ways that control flow and values were passed. A lot of important ideas were +covered and it's worth ensuring your understanding is firm. + +The only way to yield (or effectively cede control) from a coroutine is to +``await`` an object that ``yield``\ s in its ``__await__`` method. +That might sound odd to you. You might be thinking: + + 1. What about a ``yield`` directly within the coroutine function? The + coroutine function becomes an + :ref:`async generator function `, a + different beast entirely. + + 2. What about a :ref:`yield from ` within the coroutine function to a (plain) + generator? + That causes the error: ``SyntaxError: yield from not allowed in a coroutine.`` + This was intentionally designed for the sake of simplicity -- mandating only + one way of using coroutines. + Initially ``yield`` was barred as well, but was re-accepted to allow for + async generators. + Despite that, ``yield from`` and ``await`` effectively do the same thing. + +======= +Futures +======= + +A :ref:`future ` is an object meant to represent a +computation's status and result. +The term is a nod to the idea of something still to come or not yet happened, +and the object is a way to keep an eye on that something. + +A future has a few important attributes. One is its state which can be either +"pending", "cancelled" or "done". +Another is its result, which is set when the state transitions to done. +Unlike a coroutine, a future does not represent the actual computation to be +done; instead, it represents the status and result of that computation, kind of +like a status light (red, yellow or green) or indicator. + +:class:`asyncio.Task` subclasses :class:`asyncio.Future` in order to gain +these various capabilities. +The prior section said tasks store a list of callbacks, which wasn't entirely +accurate. +It's actually the ``Future`` class that implements this logic, which ``Task`` +inherits. + +Futures may also be used directly (not via tasks). +Tasks mark themselves as done when their coroutine is complete. +Futures are much more versatile and will be marked as done when you say so. +In this way, they're the flexible interface for you to make your own conditions +for waiting and resuming. + +======================== +A homemade asyncio.sleep +======================== + +We'll go through an example of how you could leverage a future to create your +own variant of asynchronous sleep (``async_sleep``) which mimics +:func:`asyncio.sleep`. + +This snippet registers a few tasks with the event loop and then awaits a +coroutine wrapped in a task: ``async_sleep(3)``. +We want that task to finish only after three seconds have elapsed, but without +preventing other tasks from running. + +:: + + async def other_work(): + print("I like work. Work work.") + + async def main(): + # Add a few other tasks to the event loop, so there's something + # to do while asynchronously sleeping. + work_tasks = [ + asyncio.create_task(other_work()), + asyncio.create_task(other_work()), + asyncio.create_task(other_work()) + ] + print( + "Beginning asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + await asyncio.create_task(async_sleep(3)) + print( + "Done asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + # asyncio.gather effectively awaits each task in the collection. + await asyncio.gather(*work_tasks) + + +Below, we use a future to enable custom control over when that task will be +marked as done. +If :meth:`future.set_result() ` (the method +responsible for marking that future as done) is never called, then this task +will never finish. +We've also enlisted the help of another task, which we'll see in a moment, that +will monitor how much time has elapsed and, accordingly, call +``future.set_result()``. + +:: + + async def async_sleep(seconds: float): + future = asyncio.Future() + time_to_wake = time.time() + seconds + # Add the watcher-task to the event loop. + watcher_task = asyncio.create_task(_sleep_watcher(future, time_to_wake)) + # Block until the future is marked as done. + await future + +Below, we'll use a rather bare object, ``YieldToEventLoop()``, to ``yield`` +from ``__await__`` in order to cede control to the event loop. +This is effectively the same as calling ``asyncio.sleep(0)``, but this approach +offers more clarity, not to mention it's somewhat cheating to use +``asyncio.sleep`` when showcasing how to implement it! + +As usual, the event loop cycles through its tasks, giving them control +and receiving control back when they pause or finish. +The ``watcher_task``, which runs the coroutine ``_sleep_watcher(...)``, will +be invoked once per full cycle of the event loop. +On each resumption, it'll check the time and if not enough has elapsed, then +it'll pause once again and hand control back to the event loop. +Eventually, enough time will have elapsed, and ``_sleep_watcher(...)`` will +mark the future as done, and then itself finish too by breaking out of the +infinite ``while`` loop. +Given this helper task is only invoked once per cycle of the event loop, +you'd be correct to note that this asynchronous sleep will sleep *at least* +three seconds, rather than exactly three seconds. +Note this is also of true of ``asyncio.sleep``. + +:: + + class YieldToEventLoop: + def __await__(self): + yield + + async def _sleep_watcher(future, time_to_wake): + while True: + if time.time() >= time_to_wake: + # This marks the future as done. + future.set_result(None) + break + else: + await YieldToEventLoop() + +Here is the full program's output: + +.. code-block:: none + + $ python custom-async-sleep.py + Beginning asynchronous sleep at time: 14:52:22. + I like work. Work work. + I like work. Work work. + I like work. Work work. + Done asynchronous sleep at time: 14:52:25. + +You might feel this implementation of asynchronous sleep was unnecessarily +convoluted. +And, well, it was. +The example was meant to showcase the versatility of futures with a simple +example that could be mimicked for more complex needs. +For reference, you could implement it without futures, like so:: + + async def simpler_async_sleep(seconds): + time_to_wake = time.time() + seconds + while True: + if time.time() >= time_to_wake: + return + else: + await YieldToEventLoop() + +But, that's all for now. Hopefully you're ready to more confidently dive into +some async programming or check out advanced topics in the +:mod:`rest of the documentation `. diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index f350141004c2db..81fc7e63f35bd7 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -1,3 +1,5 @@ +.. _how-tos: + *************** Python HOWTOs *************** @@ -11,6 +13,7 @@ Python Library Reference. :maxdepth: 1 :hidden: + a-conceptual-overview-of-asyncio.rst cporting.rst curses.rst descriptor.rst @@ -38,6 +41,7 @@ Python Library Reference. General: +* :ref:`a-conceptual-overview-of-asyncio` * :ref:`annotations-howto` * :ref:`argparse-tutorial` * :ref:`descriptorhowto` diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index b24459b5c6346f..319b2c81505f48 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -289,9 +289,9 @@ Literals * ``conversion`` is an integer: * -1: no formatting - * 115 (``ord('s')``): ``!s`` string formatting - * 114 (``ord('r')``): ``!r`` repr formatting - * 97 (``ord('a')``): ``!a`` ASCII formatting + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` formatting + * 114 (``ord('r')``): ``!r`` :func:`repr` formatting + * 115 (``ord('s')``): ``!s`` :func:`string ` formatting * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both @@ -325,14 +325,18 @@ Literals Constant(value='.3')]))])) -.. class:: TemplateStr(values) +.. class:: TemplateStr(values, /) - A t-string, comprising a series of :class:`Interpolation` and :class:`Constant` - nodes. + .. versionadded:: 3.14 + + Node representing a template string literal, comprising a series of + :class:`Interpolation` and :class:`Constant` nodes. + These nodes may be any order, and do not need to be interleaved. .. doctest:: - >>> print(ast.dump(ast.parse('t"{name} finished {place:ordinal}"', mode='eval'), indent=4)) + >>> expr = ast.parse('t"{name} finished {place:ordinal}"', mode='eval') + >>> print(ast.dump(expr, indent=4)) Expression( body=TemplateStr( values=[ @@ -349,28 +353,28 @@ Literals values=[ Constant(value='ordinal')]))])) - .. versionadded:: 3.14 - +.. class:: Interpolation(value, str, conversion, format_spec=None) -.. class:: Interpolation(value, str, conversion, format_spec) + .. versionadded:: 3.14 - Node representing a single interpolation field in a t-string. + Node representing a single interpolation field in a template string literal. * ``value`` is any expression node (such as a literal, a variable, or a function call). + This has the same meaning as ``FormattedValue.value``. * ``str`` is a constant containing the text of the interpolation expression. * ``conversion`` is an integer: * -1: no conversion - * 115: ``!s`` string conversion - * 114: ``!r`` repr conversion - * 97: ``!a`` ascii conversion + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` conversion + * 114 (``ord('r')``): ``!r`` :func:`repr` conversion + * 115 (``ord('s')``): ``!s`` :func:`string ` conversion + This has the same meaning as ``FormattedValue.conversion``. * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both ``conversion`` and ``format_spec`` can be set at the same time. - - .. versionadded:: 3.14 + This has the same meaning as ``FormattedValue.format_spec``. .. class:: List(elts, ctx) diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 32771ba72e0002..4b69e569523c58 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -75,6 +75,7 @@ Future Functions Deprecation warning is emitted if *future* is not a Future-like object and *loop* is not specified and there is no running event loop. +.. _asyncio-future-obj: Future Object ============= diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index b19ffa8213a971..f825ae92ec7471 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1193,6 +1193,7 @@ Introspection .. versionadded:: 3.4 +.. _asyncio-task-obj: Task Object =========== diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 7d368dae49dc1d..444db01390d922 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -29,6 +29,11 @@ database connection libraries, distributed task queues, etc. asyncio is often a perfect fit for IO-bound and high-level **structured** network code. +.. seealso:: + + :ref:`a-conceptual-overview-of-asyncio` + Explanation of the fundamentals of asyncio. + asyncio provides a set of **high-level** APIs to: * :ref:`run Python coroutines ` concurrently and diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index b292d828841f2f..fd397547a04437 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -501,6 +501,14 @@ The :mod:`calendar` module exports the following data attributes: >>> list(calendar.month_name) ['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] + .. caution:: + + In locales with alternative month names forms, the :data:`!month_name` sequence + may not be suitable when a month name stands by itself and not as part of a date. + For instance, in Greek and in many Slavic and Baltic languages, :data:`!month_name` + will produce the month in genitive case. Use :data:`standalone_month_name` for a form + suitable for standalone use. + .. data:: month_abbr @@ -512,6 +520,31 @@ The :mod:`calendar` module exports the following data attributes: >>> list(calendar.month_abbr) ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + .. caution:: + + In locales with alternative month names forms, the :data:`!month_abbr` sequence + may not be suitable when a month name stands by itself and not as part of a date. + Use :data:`standalone_month_abbr` for a form suitable for standalone use. + + +.. data:: standalone_month_name + + A sequence that represents the months of the year in the current locale + in the standalone form if the locale provides one. Else it is equivalent + to :data:`month_name`. + + .. versionadded:: next + + +.. data:: standalone_month_abbr + + A sequence that represents the abbreviated months of the year in the current + locale in the standalone form if the locale provides one. Else it is + equivalent to :data:`month_abbr`. + + .. versionadded:: next + + .. data:: JANUARY FEBRUARY MARCH diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index f96f2f8281f450..5932012c535b56 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1251,7 +1251,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | iso8859_3 | iso-8859-3, latin3, L3 | Esperanto, Maltese | +-----------------+--------------------------------+--------------------------------+ -| iso8859_4 | iso-8859-4, latin4, L4 | Baltic languages | +| iso8859_4 | iso-8859-4, latin4, L4 | Northern Europe | +-----------------+--------------------------------+--------------------------------+ | iso8859_5 | iso-8859-5, cyrillic | Belarusian, Bulgarian, | | | | Macedonian, Russian, Serbian | diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 09f596101b4d1e..d8dac24c8ab532 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -700,14 +700,10 @@ compiler does it. It is possible to override this behavior entirely by specifyi :attr:`~Structure._layout_` class attribute in the subclass definition; see the attribute documentation for details. -It is possible to specify the maximum alignment for the fields by setting -the :attr:`~Structure._pack_` class attribute to a positive integer. -This matches what ``#pragma pack(n)`` does in MSVC. - -It is also possible to set a minimum alignment for how the subclass itself is packed in the -same way ``#pragma align(n)`` works in MSVC. -This can be achieved by specifying a :attr:`~Structure._align_` class attribute -in the subclass definition. +It is possible to specify the maximum alignment for the fields and/or for the +structure itself by setting the class attributes :attr:`~Structure._pack_` +and/or :attr:`~Structure._align_`, respectively. +See the attribute documentation for details. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -2792,11 +2788,18 @@ fields, or any other data types containing pointer type fields. .. attribute:: _pack_ An optional small integer that allows overriding the alignment of - structure fields in the instance. :attr:`_pack_` must already be defined - when :attr:`_fields_` is assigned, otherwise it will have no effect. - Setting this attribute to 0 is the same as not setting it at all. + structure fields in the instance. + + This is only implemented for the MSVC-compatible memory layout + (see :attr:`_layout_`). - This is only implemented for the MSVC-compatible memory layout. + Setting :attr:`!_pack_` to 0 is the same as not setting it at all. + Otherwise, the value must be a positive power of two. + The effect is equivalent to ``#pragma pack(N)`` in C, except + :mod:`ctypes` may allow larger *n* than what the compiler accepts. + + :attr:`!_pack_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. deprecated-removed:: 3.14 3.19 @@ -2809,9 +2812,22 @@ fields, or any other data types containing pointer type fields. .. attribute:: _align_ - An optional small integer that allows overriding the alignment of + An optional small integer that allows increasing the alignment of the structure when being packed or unpacked to/from memory. - Setting this attribute to 0 is the same as not setting it at all. + + The value must not be negative. + The effect is equivalent to ``__attribute__((aligned(N)))`` on GCC + or ``#pragma align(N)`` on MSVC, except :mod:`ctypes` may allow + values that the compiler would reject. + + :attr:`!_align_` can only *increase* a structure's alignment + requirements. Setting it to 0 or 1 has no effect. + + Using values that are not powers of two is discouraged and may lead to + surprising behavior. + + :attr:`!_align_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. versionadded:: 3.13 diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 16ed3215bc2c1a..7010f99c54da0a 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2383,7 +2383,7 @@ locations where different offsets are used in different days of the year or where historical changes have been made to civil time. -.. class:: timezone(offset, name=None) +.. class:: timezone(offset[, name]) The *offset* argument must be specified as a :class:`timedelta` object representing the difference between the local time and UTC. It must diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index ce948a6860f02c..c55ecac340972b 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -278,7 +278,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. emu -.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') +.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n', *, color=False) Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in unified diff format. @@ -297,6 +297,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. For inputs that do not have trailing newlines, set the *lineterm* argument to ``""`` so that the output will be uniformly newline free. + Set *color* to ``True`` to enable output in color, similar to + :program:`git diff --color`. Even if enabled, it can be + :ref:`controlled using environment variables `. + The unified diff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for *fromfile*, *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally @@ -319,6 +323,10 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. See :ref:`difflib-interface` for a more detailed example. + .. versionchanged:: next + Added the *color* parameter. + + .. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n') Compare *a* and *b* (lists of bytes objects) using *dfunc*; yield a @@ -351,9 +359,9 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. seealso:: - `Pattern Matching: The Gestalt Approach `_ + `Pattern Matching: The Gestalt Approach `_ Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in `Dr. Dobb's Journal `_ in July, 1988. + was published in Dr. Dobb's Journal in July, 1988. .. _sequence-matcher: diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index ac8a911c40a860..7360f4aa804724 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1122,8 +1122,8 @@ iterations of the loop. .. opcode:: BUILD_TEMPLATE - Constructs a new :class:`~string.templatelib.Template` from a tuple - of strings and a tuple of interpolations and pushes the resulting instance + Constructs a new :class:`~string.templatelib.Template` instance from a tuple + of strings and a tuple of interpolations and pushes the resulting object onto the stack:: interpolations = STACK.pop() @@ -1135,8 +1135,8 @@ iterations of the loop. .. opcode:: BUILD_INTERPOLATION (format) - Constructs a new :class:`~string.templatelib.Interpolation` from a - value and its source expression and pushes the resulting instance onto the + Constructs a new :class:`~string.templatelib.Interpolation` instance from a + value and its source expression and pushes the resulting object onto the stack. If no conversion or format specification is present, ``format`` is set to diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 8796056b4b8722..d6d1c7a461c51c 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -14,8 +14,8 @@ The :mod:`fractions` module provides support for rational number arithmetic. -A Fraction instance can be constructed from a pair of integers, from -another rational number, or from a string. +A Fraction instance can be constructed from a pair of rational numbers, from +a single number, or from a string. .. index:: single: as_integer_ratio() @@ -25,8 +25,8 @@ another rational number, or from a string. The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance - with value equal to ``numerator/denominator`` where the denominator is positive. - If *denominator* is ``0``, it raises a :exc:`ZeroDivisionError`. + with a value equal to ``numerator/denominator``. + If *denominator* is zero, it raises a :exc:`ZeroDivisionError`. The second version requires that *number* is an instance of :class:`numbers.Rational` or has the :meth:`!as_integer_ratio` method @@ -125,7 +125,8 @@ another rational number, or from a string. .. attribute:: denominator - Denominator of the Fraction in lowest term. + Denominator of the Fraction in lowest terms. + Guaranteed to be positive. .. method:: as_integer_ratio() diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 80bd1275973f8d..857b40f3ba155c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1577,7 +1577,7 @@ are always available. They are listed here in alphabetical order. ``pow(base, exp) % mod``). The two-argument form ``pow(base, exp)`` is equivalent to using the power operator: ``base**exp``. - The arguments must have numeric types. With mixed operand types, the + When arguments are builtin numeric types with mixed operand types, the coercion rules for binary arithmetic operators apply. For :class:`int` operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 681d0b76f2a14b..57b35017072c97 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -69,11 +69,11 @@ The numeric tower .. attribute:: numerator - Abstract. + Abstract. The numerator of this rational number. .. attribute:: denominator - Abstract. + Abstract. The denominator of this rational number. .. class:: Integral diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index a14af6d3d88df2..d317ead66f9bcb 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -509,7 +509,7 @@ Module constants .. data:: SQLITE_KEYWORDS - A :class:`tuple` containing all sqlite3 keywords. + A :class:`tuple` containing all SQLite keywords. This constant is only available if Python was compiled with SQLite 3.24.0 or greater. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 90683c0b00d78a..a81a6704142dcc 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2673,9 +2673,10 @@ For example: The formatting operations described here exhibit a variety of quirks that lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). Using the newer :ref:`formatted string literals - `, the :meth:`str.format` interface, or :ref:`template strings - ($-strings) ` may help avoid these errors. + dictionaries correctly). + + Using :ref:`formatted string literals `, the :meth:`str.format` + interface, or :class:`string.Template` may help avoid these errors. Each of these alternatives provides their own trade-offs and benefits of simplicity, flexibility, and/or extensibility. diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 83e8ee2722ed8a..6336a0ec47b91e 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -200,7 +200,7 @@ syntax for format strings (although in the case of :class:`Formatter`, subclasses can define their own format string syntax). The syntax is related to that of :ref:`formatted string literals ` and :ref:`template string literals `, but it is less sophisticated -and, in particular, does not support arbitrary expressions. +and, in particular, does not support arbitrary expressions in interpolations. .. index:: single: {} (curly brackets); in string formatting @@ -799,13 +799,15 @@ Template strings ($-strings) .. note:: - The feature described here was introduced in Python 2.4. It is unrelated - to, and should not be confused with, the newer - :ref:`template strings ` and - :ref:`t-string literal syntax ` introduced in Python 3.14. - T-string literals evaluate to instances of a different - :class:`~string.templatelib.Template` class, found in the - :mod:`string.templatelib` module. + The feature described here was introduced in Python 2.4; + a simple templating method based upon regular expressions. + It predates :meth:`str.format`, :ref:`formatted string literals `, + and :ref:`template string literals `. + + It is unrelated to template string literals (t-strings), + which were introduced in Python 3.14. + These evaluate to :class:`string.templatelib.Template` objects, + found in the :mod:`string.templatelib` module. Template strings provide simpler string substitutions as described in :pep:`292`. A primary use case for template strings is for diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst index 31b90d75f411f0..19daf352bdc5fc 100644 --- a/Doc/library/string.templatelib.rst +++ b/Doc/library/string.templatelib.rst @@ -11,8 +11,8 @@ .. seealso:: * :ref:`Format strings ` - * :ref:`T-string literal syntax ` - + * :ref:`Template string literal (t-string) syntax ` + * :pep:`750` .. _template-strings: @@ -21,278 +21,311 @@ Template strings .. versionadded:: 3.14 -Template strings are a formatting mechanism that allows for deep control over -how strings are processed. You can create templates using -:ref:`t-string literal syntax `, which is identical to -:ref:`f-string syntax ` but uses a ``t`` instead of an ``f``. -While f-strings evaluate to ``str``, t-strings create a :class:`Template` -instance that gives you access to the static and interpolated (in curly braces) -parts of a string *before* they are combined. - - -.. _templatelib-template: +Template strings are a mechanism for custom string processing. +They have the full flexibility of Python's :ref:`f-strings`, +but return a :class:`Template` instance that gives access +to the static and interpolated (in curly braces) parts of a string +*before* they are combined. -Template --------- +To write a t-string, use a ``'t'`` prefix instead of an ``'f'``, like so: -The :class:`!Template` class describes the contents of a template string. +.. code-block:: pycon -:class:`!Template` instances are immutable: their attributes cannot be -reassigned. + >>> pi = 3.14 + >>> t't-strings are new in Python {pi!s}!' + Template( + strings=('t-strings are new in Python ', '.'), + interpolations=(Interpolation(3.14, 'pi', 's', ''),) + ) -.. class:: Template(*args) +Types +----- - Create a new :class:`!Template` object. +.. class:: Template - :param args: A mix of strings and :class:`Interpolation` instances in any order. - :type args: str | Interpolation + The :class:`!Template` class describes the contents of a template string. + It is immutable, meaning that attributes of a template cannot be reassigned. The most common way to create a :class:`!Template` instance is to use the - :ref:`t-string literal syntax `. This syntax is identical to that of - :ref:`f-strings ` except that it uses a ``t`` instead of an ``f``: + :ref:`template string literal syntax `. + This syntax is identical to that of :ref:`f-strings `, + except that it uses a ``t`` prefix in place of an ``f``: - >>> name = "World" - >>> template = t"Hello {name}!" + >>> cheese = 'Red Leicester' + >>> template = t"We're fresh out of {cheese}, sir." >>> type(template) - Templates ars stored as sequences of literal :attr:`~Template.strings` + Templates are stored as sequences of literal :attr:`~Template.strings` and dynamic :attr:`~Template.interpolations`. - A :attr:`~Template.values` attribute holds the interpolation values: + A :attr:`~Template.values` attribute holds the values of the interpolations: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' >>> template.strings - ('Hello ', '!') + ('Ah! We do have ', '.') >>> template.interpolations - (Interpolation('World', ...),) + (Interpolation('Camembert', ...),) >>> template.values - ('World',) + ('Camembert',) The :attr:`!strings` tuple has one more element than :attr:`!interpolations` and :attr:`!values`; the interpolations “belong” between the strings. - This may be easier to understand when tuples are aligned:: + This may be easier to understand when tuples are aligned - template.strings: ('Hello ', '!') - template.values: ( 'World', ) + .. code-block:: python - While literal syntax is the most common way to create :class:`!Template` - instances, it is also possible to create them directly using the constructor: + template.strings: ('Ah! We do have ', '.') + template.values: ( 'Camembert', ) - >>> from string.templatelib import Interpolation, Template - >>> name = "World" - >>> template = Template("Hello, ", Interpolation(name, "name"), "!") - >>> list(template) - ['Hello, ', Interpolation('World', 'name', None, ''), '!'] + .. rubric:: Attributes - If two or more consecutive strings are passed, they will be concatenated - into a single value in the :attr:`~Template.strings` attribute. For example, - the following code creates a :class:`Template` with a single final string: + .. attribute:: strings + :type: tuple[str, ...] - >>> from string.templatelib import Template - >>> template = Template("Hello ", "World", "!") - >>> template.strings - ('Hello World!',) + A :class:`tuple` of the static strings in the template. - If two or more consecutive interpolations are passed, they will be treated - as separate interpolations and an empty string will be inserted between them. - For example, the following code creates a template with empty placeholders - in the :attr:`~Template.strings` attribute: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.strings + ('Ah! We do have ', '.') - >>> from string.templatelib import Interpolation, Template - >>> template = Template(Interpolation("World", "name"), Interpolation("!", "punctuation")) - >>> template.strings - ('', '', '') + Empty strings *are* included in the tuple: - .. attribute:: strings - :type: tuple[str, ...] + >>> response = 'We do have ' + >>> cheese = 'Camembert' + >>> template = t'Ah! {response}{cheese}.' + >>> template.strings + ('Ah! ', '', '.') - A :ref:`tuple ` of the static strings in the template. + The ``strings`` tuple is never empty, and always contains one more + string than the ``interpolations`` and ``values`` tuples: - >>> name = "World" - >>> t"Hello {name}!".strings - ('Hello ', '!') + >>> t''.strings + ('',) + >>> t''.values + () + >>> t'{'cheese'}'.strings + ('', '') + >>> t'{'cheese'}'.values + ('cheese',) - Empty strings *are* included in the tuple: + .. attribute:: interpolations + :type: tuple[Interpolation, ...] - >>> name = "World" - >>> t"Hello {name}{name}!".strings - ('Hello ', '', '!') + A :class:`tuple` of the interpolations in the template. - The ``strings`` tuple is never empty, and always contains one more - string than the ``interpolations`` and ``values`` tuples: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.interpolations + (Interpolation('Camembert', 'cheese', None, ''),) - >>> t"".strings - ('',) - >>> t"".values - () - >>> t"{'cheese'}".strings - ('', '') - >>> t"{'cheese'}".values - ('cheese',) + The ``interpolations`` tuple may be empty and always contains one fewer + values than the ``strings`` tuple: - .. attribute:: interpolations - :type: tuple[Interpolation, ...] + >>> t'Red Leicester'.interpolations + () - A tuple of the interpolations in the template. + .. attribute:: values + :type: tuple[object, ...] - >>> name = "World" - >>> t"Hello {name}!".interpolations - (Interpolation('World', 'name', None, ''),) + A tuple of all interpolated values in the template. - The ``interpolations`` tuple may be empty and always contains one fewer - values than the ``strings`` tuple: + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.values + ('Camembert',) - >>> t"Hello!".interpolations - () + The ``values`` tuple always has the same length as the + ``interpolations`` tuple. It is always equivalent to + ``tuple(i.value for i in template.interpolations)``. - .. attribute:: values - :type: tuple[Any, ...] + .. rubric:: Methods - A tuple of all interpolated values in the template. + .. method:: __new__(*args: str | Interpolation) - >>> name = "World" - >>> t"Hello {name}!".values - ('World',) + While literal syntax is the most common way to create a :class:`!Template`, + it is also possible to create them directly using the constructor: - The ``values`` tuple always has the same length as the - ``interpolations`` tuple. It is equivalent to - ``tuple(i.value for i in template.interpolations)``. + >>> from string.templatelib import Interpolation, Template + >>> cheese = 'Camembert' + >>> template = Template( + ... 'Ah! We do have ', Interpolation(cheese, 'cheese'), '.' + ... ) + >>> list(template) + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] - .. describe:: iter(template) + If multiple strings are passed consecutively, they will be concatenated + into a single value in the :attr:`~Template.strings` attribute. For example, + the following code creates a :class:`Template` with a single final string: - Iterate over the template, yielding each string and - :class:`Interpolation` in order. + >>> from string.templatelib import Template + >>> template = Template('Ah! We do have ', 'Camembert', '.') + >>> template.strings + ('Ah! We do have Camembert.',) - >>> name = "World" - >>> list(t"Hello {name}!") - ['Hello ', Interpolation('World', 'name', None, ''), '!'] + If multiple interpolations are passed consecutively, they will be treated + as separate interpolations and an empty string will be inserted between them. + For example, the following code creates a template with empty placeholders + in the :attr:`~Template.strings` attribute: - Empty strings are *not* included in the iteration: + >>> from string.templatelib import Interpolation, Template + >>> template = Template( + ... Interpolation('Camembert', 'cheese'), + ... Interpolation('.', 'punctuation'), + ... ) + >>> template.strings + ('', '', '') - >>> name = "World" - >>> list(t"Hello {name}{name}") - ['Hello ', Interpolation('World', 'name', None, ''), Interpolation('World', 'name', None, '')] + .. describe:: iter(template) - .. describe:: template + other - template += other + Iterate over the template, yielding each non-empty string and + :class:`Interpolation` in the correct order: - Concatenate this template with another, returning a new - :class:`!Template` instance: + >>> cheese = 'Camembert' + >>> list(t'Ah! We do have {cheese}.') + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] - >>> name = "World" - >>> list(t"Hello " + t"there {name}!") - ['Hello there ', Interpolation('World', 'name', None, ''), '!'] + .. caution:: - Concatenation between a :class:`!Template` and a ``str`` is *not* supported. - This is because it is ambiguous whether the string should be treated as - a static string or an interpolation. If you want to concatenate a - :class:`!Template` with a string, you should either wrap the string - directly in a :class:`!Template` (to treat it as a static string) or use - an :class:`!Interpolation` (to treat it as dynamic): + Empty strings are **not** included in the iteration: - >>> from string.templatelib import Template, Interpolation - >>> template = t"Hello " - >>> # Treat "there " as a static string - >>> template += Template("there ") - >>> # Treat name as an interpolation - >>> name = "World" - >>> template += Template(Interpolation(name, "name")) - >>> list(template) - ['Hello there ', Interpolation('World', 'name', None, '')] + >>> response = 'We do have ' + >>> cheese = 'Camembert' + >>> list(t'Ah! {response}{cheese}.') # doctest: +NORMALIZE_WHITESPACE + ['Ah! ', + Interpolation('We do have ', 'response', None, ''), + Interpolation('Camembert', 'cheese', None, ''), + '.'] + .. describe:: template + other + template += other -.. class:: Interpolation(value, expression="", conversion=None, format_spec="") + Concatenate this template with another, returning a new + :class:`!Template` instance: - Create a new :class:`!Interpolation` object. + >>> cheese = 'Camembert' + >>> list(t'Ah! ' + t'We do have {cheese}.') + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] - :param value: The evaluated, in-scope result of the interpolation. - :type value: object + Concatenating a :class:`!Template` and a ``str`` is **not** supported. + This is because it is unclear whether the string should be treated as + a static string or an interpolation. + If you want to concatenate a :class:`!Template` with a string, + you should either wrap the string directly in a :class:`!Template` + (to treat it as a static string) + or use an :class:`!Interpolation` (to treat it as dynamic): - :param expression: The text of a valid Python expression, or an empty string. - :type expression: str + >>> from string.templatelib import Interpolation, Template + >>> template = t'Ah! ' + >>> # Treat 'We do have ' as a static string + >>> template += Template('We do have ') + >>> # Treat cheese as an interpolation + >>> cheese = 'Camembert' + >>> template += Template(Interpolation(cheese, 'cheese')) + >>> list(template) + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, '')] - :param conversion: The optional :ref:`conversion ` to be used, one of r, s, and a. - :type conversion: ``Literal["a", "r", "s"] | None`` - :param format_spec: An optional, arbitrary string used as the :ref:`format specification ` to present the value. - :type format_spec: str +.. class:: Interpolation The :class:`!Interpolation` type represents an expression inside a template string. + It is immutable, meaning that attributes of an interpolation cannot be reassigned. - :class:`!Interpolation` instances are immutable: their attributes cannot be - reassigned. + Interpolations support pattern matching, allowing you to match against + their attributes with the :ref:`match statement `: + + >>> from string.templatelib import Interpolation + >>> interpolation = t'{1. + 2.:.2f}'.interpolations[0] + >>> interpolation + Interpolation(3.0, '1. + 2.', None, '.2f') + >>> match interpolation: + ... case Interpolation(value, expression, conversion, format_spec): + ... print(value, expression, conversion, format_spec, sep=' | ') + ... + 3.0 | 1. + 2. | None | .2f + + .. rubric:: Attributes .. attribute:: value + :type: object - :returns: The evaluated value of the interpolation. - :type: object + The evaluated value of the interpolation. - >>> t"{1 + 2}".interpolations[0].value - 3 + >>> t'{1 + 2}'.interpolations[0].value + 3 .. attribute:: expression + :type: str - :returns: The text of a valid Python expression, or an empty string. - :type: str + The text of a valid Python expression, or an empty string. - The :attr:`~Interpolation.expression` is the original text of the - interpolation's Python expression, if the interpolation was created - from a t-string literal. Developers creating interpolations manually - should either set this to an empty string or choose a suitable valid - Python expression. + The :attr:`.expression` is the original text of the + interpolation's Python expression, if the interpolation was created + from a t-string literal. Developers creating interpolations manually + should either set this to an empty string or choose a suitable valid + Python expression. - >>> t"{1 + 2}".interpolations[0].expression - '1 + 2' + >>> t'{1 + 2}'.interpolations[0].expression + '1 + 2' .. attribute:: conversion + :type: typing.Literal['a', 'r', 's'] | None - :returns: The conversion to apply to the value, or ``None``. - :type: ``Literal["a", "r", "s"] | None`` + The conversion to apply to the value, or ``None``. - The :attr:`!Interpolation.conversion` is the optional conversion to apply - to the value: + The :attr:`!conversion` is the optional conversion to apply + to the value: - >>> t"{1 + 2!a}".interpolations[0].conversion - 'a' + >>> t'{1 + 2!a}'.interpolations[0].conversion + 'a' - .. note:: + .. note:: Unlike f-strings, where conversions are applied automatically, the expected behavior with t-strings is that code that *processes* the :class:`!Template` will decide how to interpret and whether to apply - the :attr:`!Interpolation.conversion`. + the :attr:`!conversion`. + For convenience, the :func:`convert` function can be used to mimic + f-string conversion semantics. .. attribute:: format_spec + :type: str - :returns: The format specification to apply to the value. - :type: str + The format specification to apply to the value. - The :attr:`!Interpolation.format_spec` is an optional, arbitrary string - used as the format specification to present the value: + The :attr:`!format_spec` is an optional, arbitrary string + used as the format specification to present the value: - >>> t"{1 + 2:.2f}".interpolations[0].format_spec - '.2f' + >>> t'{1 + 2:.2f}'.interpolations[0].format_spec + '.2f' - .. note:: + .. note:: Unlike f-strings, where format specifications are applied automatically via the :func:`format` protocol, the expected behavior with - t-strings is that code that *processes* the :class:`!Template` will + t-strings is that code that *processes* the interpolation will decide how to interpret and whether to apply the format specification. - As a result, :attr:`!Interpolation.format_spec` values in - :class:`!Template` instances can be arbitrary strings, even those that - do not necessarily conform to the rules of Python's :func:`format` - protocol. + As a result, :attr:`!format_spec` values in interpolations + can be arbitrary strings, + including those that do not conform to the :func:`format` protocol. - Interpolations support pattern matching, allowing you to match against - their attributes with the :ref:`match statement `: + .. rubric:: Methods - >>> from string.templatelib import Interpolation - >>> interpolation = Interpolation(3.0, "1 + 2", None, ".2f") - >>> match interpolation: - ... case Interpolation(value, expression, conversion, format_spec): - ... print(value, expression, conversion, format_spec) - ... - 3.0 1 + 2 None .2f + .. method:: __new__(value: object, \ + expression: str, \ + conversion: typing.Literal['a', 'r', 's'] | None = None, \ + format_spec: str = '') + + Create a new :class:`!Interpolation` object from component parts. + + :param value: The evaluated, in-scope result of the interpolation. + :param expression: The text of a valid Python expression, + or an empty string. + :param conversion: The :ref:`conversion ` to be used, + one of ``None``, ``'a'``, ``'r'``, or ``'s'``. + :param format_spec: An optional, arbitrary string used as the + :ref:`format specification ` to present the value. Helper functions @@ -306,8 +339,8 @@ Helper functions Three conversion flags are currently supported: - * ``'s'`` which calls :func:`str` on the value, - * ``'r'`` which calls :func:`repr`, and - * ``'a'`` which calls :func:`ascii`. + * ``'s'`` which calls :func:`str` on the value (like ``!s``), + * ``'r'`` which calls :func:`repr` (like ``!r``), and + * ``'a'`` which calls :func:`ascii` (like ``!a``). If the conversion flag is ``None``, *obj* is returned unchanged. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 52f0af31c68726..771e0f2709a4aa 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -2152,11 +2152,16 @@ always available. Unless explicitly noted otherwise, all variables are read-only The default hook formats :attr:`!err_msg` and :attr:`!object` as: ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message - if :attr:`!err_msg` is ``None``. + if :attr:`!err_msg` is ``None``. Similar to the :mod:`traceback` module, + this adds color to exceptions by default. This can be disabled using + :ref:`environment variables `. :func:`sys.unraisablehook` can be overridden to control how unraisable exceptions are handled. + .. versionchanged:: next + Exceptions are now printed with colorful text. + .. seealso:: :func:`excepthook` which handles uncaught exceptions. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index df9be68bf4f69d..b05c0a312dbe34 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -935,7 +935,7 @@ These constants are used as parameters for :func:`clock_getres` and .. data:: CLOCK_TAI - `International Atomic Time `_ + `International Atomic Time `_ The system must have a current leap second table in order for this to give the correct answer. PTP or NTP software can maintain a leap second table. diff --git a/Doc/library/tulip_coro.dia b/Doc/library/tulip_coro.dia deleted file mode 100644 index 70a33e3c00cf6e..00000000000000 Binary files a/Doc/library/tulip_coro.dia and /dev/null differ diff --git a/Doc/library/tulip_coro.png b/Doc/library/tulip_coro.png deleted file mode 100644 index aad41c93015d09..00000000000000 Binary files a/Doc/library/tulip_coro.png and /dev/null differ diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 69df09c779592a..9dbac8ce75d489 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -45,15 +45,15 @@ provides backports of these new features to older versions of Python. .. seealso:: - `"Typing cheat sheet" `_ + `Typing cheat sheet `_ A quick overview of type hints (hosted at the mypy docs) - "Type System Reference" section of `the mypy docs `_ + Type System Reference section of `the mypy docs `_ The Python typing system is standardised via PEPs, so this reference should broadly apply to most Python type checkers. (Some parts may still be specific to mypy.) - `"Static Typing with Python" `_ + `Static Typing with Python `_ Type-checker-agnostic documentation written by the community detailing type system features, useful typing related tools and typing best practices. @@ -64,7 +64,7 @@ Specification for the Python Type System ======================================== The canonical, up-to-date specification of the Python type system can be -found at `"Specification for the Python type system" `_. +found at `Specification for the Python type system `_. .. _type-aliases: @@ -2573,7 +2573,7 @@ types. at runtime as soon as the class has been created. Monkey-patching attributes onto a runtime-checkable protocol will still work, but will have no impact on :func:`isinstance` checks comparing objects to the - protocol. See :ref:`"What's new in Python 3.12" ` + protocol. See :ref:`What's new in Python 3.12 ` for more details. @@ -3357,7 +3357,7 @@ Introspection helpers with ``T``, unless *include_extras* is set to ``True`` (see :class:`Annotated` for more information). - See also :func:`inspect.get_annotations`, a lower-level function that + See also :func:`annotationlib.get_annotations`, a lower-level function that returns annotations more directly. .. note:: diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 091562cc9aef98..91f90a0726aa93 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -639,7 +639,7 @@ the *new_callable* argument to :func:`patch`. This is either ``None`` (if the mock hasn't been called), or the arguments that the mock was last called with. This will be in the form of a tuple: the first member, which can also be accessed through - the ``args`` property, is any ordered arguments the mock was + the ``args`` property, is any positional arguments the mock was called with (or an empty tuple) and the second member, which can also be accessed through the ``kwargs`` property, is any keyword arguments (or an empty dictionary). diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index e514b98fc5d553..5f796578eaa64e 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -837,7 +837,7 @@ The following attribute and methods should only be used by classes derived from 1. a :class:`Request` object, #. a file-like object with the HTTP error body, #. the three-digit code of the error, as a string, - #. the user-visible explanation of the code, as as string, and + #. the user-visible explanation of the code, as a string, and #. the headers of the error, as a mapping object. Return values and exceptions raised should be the same as those of diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 00bafd1be4bd0c..05e061cc697778 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -458,7 +458,7 @@ Available Functions lower.one_way(**kw) This makes the warning refer to both the ``example.lower.one_way()`` and - ``package.higher.another_way()`` call sites only from calling code living + ``example.higher.another_way()`` call sites only from calling code living outside of ``example`` package. *source*, if supplied, is the destroyed object which emitted a diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 7ac4d8587ce7d5..4e49a49c08167a 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -386,7 +386,7 @@ have ambiguous semantics. It is not possible to mix :keyword:`except` and :keyword:`!except*` in the same :keyword:`try`. -:keyword:`break`, :keyword:`continue` and :keyword:`return` +The :keyword:`break`, :keyword:`continue`, and :keyword:`return` statements cannot appear in an :keyword:`!except*` clause. diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index cf241829b71120..e320eedfa67a27 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -599,9 +599,9 @@ The allowed prefixes are: See the linked sections for details on each type. -Prefixes are case-insensitive (for example, ``B`` works the same as ``b``). -The ``r`` prefix can be combined with ``f``, ``t`` or ``b``, so ``fr``, -``rf``, ``tr``, ``rt``, ``br`` and ``rb`` are also valid prefixes. +Prefixes are case-insensitive (for example, '``B``' works the same as '``b``'). +The '``r``' prefix can be combined with '``f``', '``t``' or '``b``', so '``fr``', +'``rf``', '``tr``', '``rt``', '``br``', and '``rb``' are also valid prefixes. .. versionadded:: 3.3 The ``'rb'`` prefix of raw bytes literals has been added as a synonym @@ -661,7 +661,7 @@ quote. Escape sequences ---------------- -Unless an ``'r'`` or ``'R'`` prefix is present, escape sequences in string and +Unless an '``r``' or '``R``' prefix is present, escape sequences in string and bytes literals are interpreted according to rules similar to those used by Standard C. The recognized escape sequences are: @@ -852,7 +852,7 @@ unrecognized escapes. Bytes literals -------------- -:dfn:`Bytes literals` are always prefixed with ``'b'`` or ``'B'``; they produce an +:dfn:`Bytes literals` are always prefixed with '``b``' or '``B``'; they produce an instance of the :class:`bytes` type instead of the :class:`str` type. They may only contain ASCII characters; bytes with a numeric value of 128 or greater must be expressed with escape sequences (typically @@ -878,8 +878,8 @@ Similarly, a zero byte must be expressed using an escape sequence (typically Raw string literals ------------------- -Both string and bytes literals may optionally be prefixed with a letter ``'r'`` -or ``'R'``; such constructs are called :dfn:`raw string literals` +Both string and bytes literals may optionally be prefixed with a letter '``r``' +or '``R``'; such constructs are called :dfn:`raw string literals` and :dfn:`raw bytes literals` respectively and treat backslashes as literal characters. As a result, in raw string literals, :ref:`escape sequences ` @@ -923,7 +923,7 @@ f-strings .. versionadded:: 3.6 A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with ``f`` or ``F``. These strings may contain +that is prefixed with '``f``' or '``F``'. These strings may contain replacement fields, which are expressions delimited by curly braces ``{}``. While other string literals always have a constant value, formatted strings are really expressions evaluated at run time. @@ -1089,37 +1089,37 @@ t-strings .. versionadded:: 3.14 A :dfn:`template string literal` or :dfn:`t-string` is a string literal -that is prefixed with ``t`` or ``T``. These strings follow the same -syntax and evaluation rules as :ref:`formatted string literals `, with -the following differences: +that is prefixed with '``t``' or '``T``'. +These strings follow the same syntax and evaluation rules as +:ref:`formatted string literals `, with the following differences: -- Rather than evaluating to a ``str`` object, t-strings evaluate to a - :class:`~string.templatelib.Template` object from the - :mod:`string.templatelib` module. +* Rather than evaluating to a ``str`` object, template string literals evaluate + to a :class:`string.templatelib.Template` object. -- The :func:`format` protocol is not used. Instead, the format specifier and - conversions (if any) are passed to a new :class:`~string.templatelib.Interpolation` - object that is created for each evaluated expression. It is up to code that - processes the resulting :class:`~string.templatelib.Template` object to - decide how to handle format specifiers and conversions. +* The :func:`format` protocol is not used. + Instead, the format specifier and conversions (if any) are passed to + a new :class:`~string.templatelib.Interpolation` object that is created + for each evaluated expression. + It is up to code that processes the resulting :class:`~string.templatelib.Template` + object to decide how to handle format specifiers and conversions. -- Format specifiers containing nested replacement fields are evaluated eagerly, +* Format specifiers containing nested replacement fields are evaluated eagerly, prior to being passed to the :class:`~string.templatelib.Interpolation` object. For instance, an interpolation of the form ``{amount:.{precision}f}`` will - evaluate the expression ``{precision}`` before setting the ``format_spec`` - attribute of the resulting :class:`!Interpolation` object; if ``precision`` - is (for example) ``2``, the resulting format specifier will be ``'.2f'``. - -- When the equal sign ``'='`` is provided in an interpolation expression, the - resulting :class:`~string.templatelib.Template` object will have the expression - text along with a ``'='`` character placed in its - :attr:`~string.templatelib.Template.strings` attribute. The - :attr:`~string.templatelib.Template.interpolations` attribute will also - contain an ``Interpolation`` instance for the expression. By default, the - :attr:`~string.templatelib.Interpolation.conversion` attribute will be set to - ``'r'`` (that is, :func:`repr`), unless there is a conversion explicitly - specified (in which case it overrides the default) or a format specifier is - provided (in which case, the ``conversion`` defaults to ``None``). + evaluate the inner expression ``{precision}`` to determine the value of the + ``format_spec`` attribute. + If ``precision`` were to be ``2``, the resulting format specifier + would be ``'.2f'``. + +* When the equals sign ``'='`` is provided in an interpolation expression, + the text of the expression is appended to the literal string that precedes + the relevant interpolation. + This includes the equals sign and any surrounding whitespace. + The :class:`!Interpolation` instance for the expression will be created as + normal, except that :attr:`~string.templatelib.Interpolation.conversion` will + be set to '``r``' (:func:`repr`) by default. + If an explicit conversion or format specifier are provided, + this will override the default behaviour. .. _numbers: diff --git a/Doc/requirements.txt b/Doc/requirements.txt index a2960ea9aa0203..7b7286429a1041 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -11,7 +11,7 @@ sphinx~=8.2.0 blurb -sphinxext-opengraph~=0.10.0 +sphinxext-opengraph~=0.12.0 sphinx-notfound-page~=1.0.0 # The theme used by the documentation is stored separately, so we need diff --git a/Doc/tools/check-epub.py b/Doc/tools/check-epub.py index 693dc239c8ad58..6a10096c117542 100644 --- a/Doc/tools/check-epub.py +++ b/Doc/tools/check-epub.py @@ -1,24 +1,30 @@ -import sys from pathlib import Path +CPYTHON_ROOT = Path( + __file__, # cpython/Doc/tools/check-epub.py + '..', # cpython/Doc/tools + '..', # cpython/Doc + '..', # cpython +).resolve() +EPUBCHECK_PATH = CPYTHON_ROOT / 'Doc' / 'epubcheck.txt' -def main() -> int: - wrong_directory_msg = "Must run this script from the repo root" - if not Path("Doc").exists() or not Path("Doc").is_dir(): - raise RuntimeError(wrong_directory_msg) - - with Path("Doc/epubcheck.txt").open(encoding="UTF-8") as f: - messages = [message.split(" - ") for message in f.read().splitlines()] - fatal_errors = [message for message in messages if message[0] == "FATAL"] +def main() -> int: + lines = EPUBCHECK_PATH.read_text(encoding='utf-8').splitlines() + fatal_errors = [line for line in lines if line.startswith('FATAL')] if fatal_errors: - print("\nError: must not contain fatal errors:\n") - for error in fatal_errors: - print(" - ".join(error)) + err_count = len(fatal_errors) + s = 's' * (err_count != 1) + print() + print(f'Error: epubcheck reported {err_count} fatal error{s}:') + print() + print('\n'.join(fatal_errors)) + return 1 - return len(fatal_errors) + print('Success: no fatal errors found.') + return 0 -if __name__ == "__main__": - sys.exit(main()) +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index ea546c6a29df44..a00f06cf46c41a 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -95,11 +95,11 @@ Some examples:: >>> repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" -The :mod:`string` module also contains support for so-called -:ref:`$-strings ` that offer yet another way to -substitute values into strings, using placeholders like ``$x`` and replacing -them with values from a dictionary. This syntax is easy to use, although -it offers much less control of the formatting. +The :mod:`string` module contains support for a simple templating approach +based upon regular expressions, via :class:`string.Template`. +This offers yet another way to substitute values into strings, +using placeholders like ``$x`` and replacing them with values from a dictionary. +This syntax is easy to use, although it offers much less control for formatting. .. index:: single: formatted string literal diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 6c00a0fcb83fee..9f01b52f1aff3b 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -183,7 +183,7 @@ Other language changes compatibility between versions of Python, ensure that an explicit ``encoding`` argument is always provided. The :ref:`opt-in encoding warning ` can be used to identify code that may be affected by this change. - The special special ``encoding='locale'`` argument uses the current locale + The special ``encoding='locale'`` argument uses the current locale encoding, and has been supported since Python 3.10. To retain the previous behaviour, Python's UTF-8 mode may be disabled with @@ -200,6 +200,10 @@ Other language changes * Several error messages incorrectly using the term "argument" have been corrected. (Contributed by Stan Ulbrych in :gh:`133382`.) +* Unraisable exceptions are now highlighted with color by default. This can be + controlled by :ref:`environment variables `. + (Contributed by Peter Bierma in :gh:`134170`.) + New modules =========== @@ -225,6 +229,14 @@ dbm difflib ------- + .. _whatsnew315-color-difflib: + +* Introduced the optional *color* parameter to :func:`difflib.unified_diff`, + enabling color output similar to :program:`git diff`. + This can be controlled by :ref:`environment variables + `. + (Contributed by Douglas Thor in :gh:`133725`.) + * Improved the styling of HTML diff pages generated by the :class:`difflib.HtmlDiff` class, and migrated the output to the HTML5 standard. (Contributed by Jiahao Li in :gh:`134580`.) @@ -414,6 +426,15 @@ ctypes (Contributed by Bénédikt Tran in :gh:`133866`.) +glob +---- + +* Removed the undocumented :func:`!glob.glob0` and :func:`!glob.glob1` + functions, which have been deprecated since Python 3.13. Use + :func:`glob.glob` and pass a directory to its *root_dir* argument instead. + (Contributed by Barney Gale in :gh:`137466`.) + + http.server ----------- @@ -477,6 +498,15 @@ typing or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) +* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises + a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles + type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` + as it was incorrectly infered in runtime before. + (Contributed by Nikita Sobolev in :gh:`137191`.) + wave ---- @@ -548,6 +578,18 @@ Deprecated C APIs signed integer type of the same size is now deprecated. (Contributed by Serhiy Storchaka in :gh:`132629`.) +* Deprecate :c:member:`~PyComplexObject.cval` field of the the + :c:type:`PyComplexObject` type. + Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` + to convert a Python complex number to/from the C :c:type:`Py_complex` + representation. + (Contributed by Sergey B Kirpichev in :gh:`128813`.) + +* Functions :c:func:`_Py_c_sum`, :c:func:`_Py_c_diff`, :c:func:`_Py_c_neg`, + :c:func:`_Py_c_prod`, :c:func:`_Py_c_quot`, :c:func:`_Py_c_pow` and + :c:func:`_Py_c_abs` are :term:`soft deprecated`. + (Contributed by Sergey B Kirpichev in :gh:`128813`.) + .. Add C API deprecations above alphabetically, not here at the end. Removed C APIs diff --git a/Include/Python.h b/Include/Python.h index 3f49b78947c9a6..b6ee500aada791 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -16,6 +16,7 @@ // Include standard header files +// When changing these files, remember to update Doc/extending/extending.rst. #include // assert() #include // uintptr_t #include // INT_MAX diff --git a/Include/cpython/complexobject.h b/Include/cpython/complexobject.h index fbdc6a91fe895c..58da80140dc4c9 100644 --- a/Include/cpython/complexobject.h +++ b/Include/cpython/complexobject.h @@ -7,7 +7,8 @@ typedef struct { double imag; } Py_complex; -// Operations on complex numbers. +/* Operations on complex numbers (soft deprecated + since Python 3.15). */ PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index be582122118e44..e13b2b373c47b1 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -198,6 +198,9 @@ struct _ts { PyObject *current_executor; + /* Internal to the JIT */ + struct _PyExitData *jit_exit; + uint64_t dict_global_version; /* Used to store/retrieve `threading.local` keys/values for this thread */ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 5e7dda3a3715a1..ca4744789f1c23 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -891,7 +891,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d_parameter_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(date)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(days)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(debug)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder)); @@ -998,6 +1000,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hours)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); @@ -1092,8 +1095,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minutes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module)); @@ -1204,6 +1209,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(script)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(security_attributes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable)); @@ -1267,9 +1273,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(text)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(threading)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(throw)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(time)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeout)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timer)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(times)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timespec)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timestamp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeunit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); @@ -1301,6 +1310,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(week)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(weekday)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(weeks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(which)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(who)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(withdata)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 6908cbf78f349e..47bb6fcecc1ba6 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -382,7 +382,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(d_parameter_type) STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) + STRUCT_FOR_ID(date) STRUCT_FOR_ID(day) + STRUCT_FOR_ID(days) STRUCT_FOR_ID(debug) STRUCT_FOR_ID(decode) STRUCT_FOR_ID(decoder) @@ -489,6 +491,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(hour) + STRUCT_FOR_ID(hours) STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) @@ -583,8 +586,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) STRUCT_FOR_ID(microsecond) + STRUCT_FOR_ID(microseconds) STRUCT_FOR_ID(milliseconds) STRUCT_FOR_ID(minute) + STRUCT_FOR_ID(minutes) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) STRUCT_FOR_ID(module) @@ -695,6 +700,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(scheduler) STRUCT_FOR_ID(script) STRUCT_FOR_ID(second) + STRUCT_FOR_ID(seconds) STRUCT_FOR_ID(security_attributes) STRUCT_FOR_ID(seek) STRUCT_FOR_ID(seekable) @@ -758,9 +764,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(text) STRUCT_FOR_ID(threading) STRUCT_FOR_ID(throw) + STRUCT_FOR_ID(time) STRUCT_FOR_ID(timeout) STRUCT_FOR_ID(timer) STRUCT_FOR_ID(times) + STRUCT_FOR_ID(timespec) + STRUCT_FOR_ID(timestamp) STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(timeunit) STRUCT_FOR_ID(top) @@ -792,6 +801,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(wbits) STRUCT_FOR_ID(week) STRUCT_FOR_ID(weekday) + STRUCT_FOR_ID(weeks) STRUCT_FOR_ID(which) STRUCT_FOR_ID(who) STRUCT_FOR_ID(withdata) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 542a75617b4d3c..7cb5bce546ac74 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -88,6 +88,7 @@ struct _ceval_runtime_state { struct trampoline_api_st trampoline_api; FILE *map_file; Py_ssize_t persist_after_fork; + _PyFrameEvalFunction prev_eval_frame; #else int _not_used; #endif @@ -943,6 +944,7 @@ struct _is { bool jit; struct _PyExecutorObject *executor_list_head; struct _PyExecutorObject *executor_deletion_list_head; + struct _PyExecutorObject *cold_executor; int executor_deletion_list_remaining_capacity; size_t trace_run_counter; _rare_events rare_events; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 347d9762f26bff..a570ae684379a4 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -281,6 +281,7 @@ Known values: Python 3.15a1 3651 (Simplify LOAD_CONST) Python 3.15a1 3652 (Virtual iterators) Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) + Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.16 will start with 3700 @@ -294,7 +295,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3653 +#define PYC_MAGIC_NUMBER 3654 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index dd1bf2d1d2b51a..7f468bbb932184 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1028,10 +1028,9 @@ enum InstructionFormat { #define HAS_ESCAPES_FLAG (512) #define HAS_EXIT_FLAG (1024) #define HAS_PURE_FLAG (2048) -#define HAS_PASSTHROUGH_FLAG (4096) -#define HAS_OPARG_AND_1_FLAG (8192) -#define HAS_ERROR_NO_POP_FLAG (16384) -#define HAS_NO_SAVE_IP_FLAG (32768) +#define HAS_ERROR_NO_POP_FLAG (4096) +#define HAS_NO_SAVE_IP_FLAG (8192) +#define HAS_PERIODIC_FLAG (16384) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1044,10 +1043,9 @@ enum InstructionFormat { #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) #define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) -#define OPCODE_HAS_PASSTHROUGH(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PASSTHROUGH_FLAG)) -#define OPCODE_HAS_OPARG_AND_1(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_OPARG_AND_1_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) +#define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1218,14 +1216,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, - [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_FAST_CHECK] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG }, [LOAD_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SMALL_INT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1250,7 +1248,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [POP_TOP] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1347,27 +1345,27 @@ _PyOpcode_macro_expansion[256] = { [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 10, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_GENERAL] = { .nuops = 7, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 } } }, [CALL_LIST_APPEND] = { .nuops = 4, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_PY_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_STR_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_TUPLE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, + [CALL_STR_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_TUPLE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_TYPE_1] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, OPARG_SIMPLE, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8b7f12bf03d624..1571e19a35032e 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -67,8 +67,9 @@ typedef struct { #endif } _PyUOpInstruction; -typedef struct { +typedef struct _PyExitData { uint32_t target; + uint16_t index; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -354,6 +355,16 @@ PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); +static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) +{ + _PyExitData *exit0 = exit - exit->index; + return (_PyExecutorObject *)(((char *)exit0) - offsetof(_PyExecutorObject, exits)); +} + +extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void); + +PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit); + static inline int is_terminator(const _PyUOpInstruction *uop) { int opcode = uop->opcode; @@ -363,6 +374,8 @@ static inline int is_terminator(const _PyUOpInstruction *uop) ); } +extern void _PyExecutor_Free(_PyExecutorObject *self); + PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); #ifdef _Py_TIER2 extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 3e41e2fd1569ca..c31c33657002ec 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -111,6 +111,8 @@ extern "C" { _Py_atomic_load_ullong_relaxed(&value) #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) +#define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -159,6 +161,8 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) +#define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index da2ed7422c9deb..70a54f5273e446 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -889,7 +889,9 @@ extern "C" { INIT_ID(d_parameter_type), \ INIT_ID(data), \ INIT_ID(database), \ + INIT_ID(date), \ INIT_ID(day), \ + INIT_ID(days), \ INIT_ID(debug), \ INIT_ID(decode), \ INIT_ID(decoder), \ @@ -996,6 +998,7 @@ extern "C" { INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(hour), \ + INIT_ID(hours), \ INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ @@ -1090,8 +1093,10 @@ extern "C" { INIT_ID(metadata), \ INIT_ID(method), \ INIT_ID(microsecond), \ + INIT_ID(microseconds), \ INIT_ID(milliseconds), \ INIT_ID(minute), \ + INIT_ID(minutes), \ INIT_ID(mod), \ INIT_ID(mode), \ INIT_ID(module), \ @@ -1202,6 +1207,7 @@ extern "C" { INIT_ID(scheduler), \ INIT_ID(script), \ INIT_ID(second), \ + INIT_ID(seconds), \ INIT_ID(security_attributes), \ INIT_ID(seek), \ INIT_ID(seekable), \ @@ -1265,9 +1271,12 @@ extern "C" { INIT_ID(text), \ INIT_ID(threading), \ INIT_ID(throw), \ + INIT_ID(time), \ INIT_ID(timeout), \ INIT_ID(timer), \ INIT_ID(times), \ + INIT_ID(timespec), \ + INIT_ID(timestamp), \ INIT_ID(timetuple), \ INIT_ID(timeunit), \ INIT_ID(top), \ @@ -1299,6 +1308,7 @@ extern "C" { INIT_ID(wbits), \ INIT_ID(week), \ INIT_ID(weekday), \ + INIT_ID(weeks), \ INIT_ID(which), \ INIT_ID(who), \ INIT_ID(withdata), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b1f411945e7856..89444f4fb83b94 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1316,10 +1316,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(date); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(day); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(days); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(debug); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1744,6 +1752,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(hours); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(id); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2120,6 +2132,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(microseconds); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(milliseconds); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2128,6 +2144,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(minutes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mod); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2568,6 +2588,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(seconds); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(security_attributes); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2820,6 +2844,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(time); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timeout); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2832,6 +2860,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timespec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timestamp); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timetuple); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2956,6 +2992,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(weeks); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(which); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index a9432401525ebb..749369a40aecdd 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -76,104 +76,107 @@ extern "C" { #define _CHECK_METHOD_VERSION_KW 352 #define _CHECK_PEP_523 353 #define _CHECK_PERIODIC 354 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 -#define _CHECK_RECURSION_REMAINING 356 -#define _CHECK_STACK_SPACE 357 -#define _CHECK_STACK_SPACE_OPERAND 358 -#define _CHECK_VALIDITY 359 -#define _COMPARE_OP 360 -#define _COMPARE_OP_FLOAT 361 -#define _COMPARE_OP_INT 362 -#define _COMPARE_OP_STR 363 -#define _CONTAINS_OP 364 -#define _CONTAINS_OP_DICT 365 -#define _CONTAINS_OP_SET 366 +#define _CHECK_PERIODIC_AT_END 355 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 356 +#define _CHECK_RECURSION_REMAINING 357 +#define _CHECK_STACK_SPACE 358 +#define _CHECK_STACK_SPACE_OPERAND 359 +#define _CHECK_VALIDITY 360 +#define _COLD_EXIT 361 +#define _COMPARE_OP 362 +#define _COMPARE_OP_FLOAT 363 +#define _COMPARE_OP_INT 364 +#define _COMPARE_OP_STR 365 +#define _CONTAINS_OP 366 +#define _CONTAINS_OP_DICT 367 +#define _CONTAINS_OP_SET 368 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 367 -#define _COPY_1 368 -#define _COPY_2 369 -#define _COPY_3 370 +#define _COPY 369 +#define _COPY_1 370 +#define _COPY_2 371 +#define _COPY_3 372 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 371 +#define _CREATE_INIT_FRAME 373 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 372 +#define _DEOPT 374 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 373 -#define _DO_CALL_FUNCTION_EX 374 -#define _DO_CALL_KW 375 +#define _DO_CALL 375 +#define _DO_CALL_FUNCTION_EX 376 +#define _DO_CALL_KW 377 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 376 +#define _ERROR_POP_N 378 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 377 -#define _EXPAND_METHOD_KW 378 -#define _FATAL_ERROR 379 +#define _EXPAND_METHOD 379 +#define _EXPAND_METHOD_KW 380 +#define _FATAL_ERROR 381 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 380 -#define _FOR_ITER_GEN_FRAME 381 -#define _FOR_ITER_TIER_TWO 382 +#define _FOR_ITER 382 +#define _FOR_ITER_GEN_FRAME 383 +#define _FOR_ITER_TIER_TWO 384 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 383 -#define _GUARD_CALLABLE_ISINSTANCE 384 -#define _GUARD_CALLABLE_LEN 385 -#define _GUARD_CALLABLE_LIST_APPEND 386 -#define _GUARD_CALLABLE_STR_1 387 -#define _GUARD_CALLABLE_TUPLE_1 388 -#define _GUARD_CALLABLE_TYPE_1 389 -#define _GUARD_DORV_NO_DICT 390 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 391 -#define _GUARD_GLOBALS_VERSION 392 -#define _GUARD_IS_FALSE_POP 393 -#define _GUARD_IS_NONE_POP 394 -#define _GUARD_IS_NOT_NONE_POP 395 -#define _GUARD_IS_TRUE_POP 396 -#define _GUARD_KEYS_VERSION 397 -#define _GUARD_NOS_DICT 398 -#define _GUARD_NOS_FLOAT 399 -#define _GUARD_NOS_INT 400 -#define _GUARD_NOS_LIST 401 -#define _GUARD_NOS_NOT_NULL 402 -#define _GUARD_NOS_NULL 403 -#define _GUARD_NOS_OVERFLOWED 404 -#define _GUARD_NOS_TUPLE 405 -#define _GUARD_NOS_UNICODE 406 -#define _GUARD_NOT_EXHAUSTED_LIST 407 -#define _GUARD_NOT_EXHAUSTED_RANGE 408 -#define _GUARD_NOT_EXHAUSTED_TUPLE 409 -#define _GUARD_THIRD_NULL 410 -#define _GUARD_TOS_ANY_SET 411 -#define _GUARD_TOS_DICT 412 -#define _GUARD_TOS_FLOAT 413 -#define _GUARD_TOS_INT 414 -#define _GUARD_TOS_LIST 415 -#define _GUARD_TOS_OVERFLOWED 416 -#define _GUARD_TOS_SLICE 417 -#define _GUARD_TOS_TUPLE 418 -#define _GUARD_TOS_UNICODE 419 -#define _GUARD_TYPE_VERSION 420 -#define _GUARD_TYPE_VERSION_AND_LOCK 421 +#define _GUARD_BINARY_OP_EXTEND 385 +#define _GUARD_CALLABLE_ISINSTANCE 386 +#define _GUARD_CALLABLE_LEN 387 +#define _GUARD_CALLABLE_LIST_APPEND 388 +#define _GUARD_CALLABLE_STR_1 389 +#define _GUARD_CALLABLE_TUPLE_1 390 +#define _GUARD_CALLABLE_TYPE_1 391 +#define _GUARD_DORV_NO_DICT 392 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 393 +#define _GUARD_GLOBALS_VERSION 394 +#define _GUARD_IS_FALSE_POP 395 +#define _GUARD_IS_NONE_POP 396 +#define _GUARD_IS_NOT_NONE_POP 397 +#define _GUARD_IS_TRUE_POP 398 +#define _GUARD_KEYS_VERSION 399 +#define _GUARD_NOS_DICT 400 +#define _GUARD_NOS_FLOAT 401 +#define _GUARD_NOS_INT 402 +#define _GUARD_NOS_LIST 403 +#define _GUARD_NOS_NOT_NULL 404 +#define _GUARD_NOS_NULL 405 +#define _GUARD_NOS_OVERFLOWED 406 +#define _GUARD_NOS_TUPLE 407 +#define _GUARD_NOS_UNICODE 408 +#define _GUARD_NOT_EXHAUSTED_LIST 409 +#define _GUARD_NOT_EXHAUSTED_RANGE 410 +#define _GUARD_NOT_EXHAUSTED_TUPLE 411 +#define _GUARD_THIRD_NULL 412 +#define _GUARD_TOS_ANY_SET 413 +#define _GUARD_TOS_DICT 414 +#define _GUARD_TOS_FLOAT 415 +#define _GUARD_TOS_INT 416 +#define _GUARD_TOS_LIST 417 +#define _GUARD_TOS_OVERFLOWED 418 +#define _GUARD_TOS_SLICE 419 +#define _GUARD_TOS_TUPLE 420 +#define _GUARD_TOS_UNICODE 421 +#define _GUARD_TYPE_VERSION 422 +#define _GUARD_TYPE_VERSION_AND_LOCK 423 +#define _HANDLE_PENDING_AND_DEOPT 424 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 422 -#define _INIT_CALL_PY_EXACT_ARGS 423 -#define _INIT_CALL_PY_EXACT_ARGS_0 424 -#define _INIT_CALL_PY_EXACT_ARGS_1 425 -#define _INIT_CALL_PY_EXACT_ARGS_2 426 -#define _INIT_CALL_PY_EXACT_ARGS_3 427 -#define _INIT_CALL_PY_EXACT_ARGS_4 428 -#define _INSERT_NULL 429 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 425 +#define _INIT_CALL_PY_EXACT_ARGS 426 +#define _INIT_CALL_PY_EXACT_ARGS_0 427 +#define _INIT_CALL_PY_EXACT_ARGS_1 428 +#define _INIT_CALL_PY_EXACT_ARGS_2 429 +#define _INIT_CALL_PY_EXACT_ARGS_3 430 +#define _INIT_CALL_PY_EXACT_ARGS_4 431 +#define _INSERT_NULL 432 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -183,177 +186,177 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 430 +#define _IS_NONE 433 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 431 -#define _ITER_CHECK_RANGE 432 -#define _ITER_CHECK_TUPLE 433 -#define _ITER_JUMP_LIST 434 -#define _ITER_JUMP_RANGE 435 -#define _ITER_JUMP_TUPLE 436 -#define _ITER_NEXT_LIST 437 -#define _ITER_NEXT_LIST_TIER_TWO 438 -#define _ITER_NEXT_RANGE 439 -#define _ITER_NEXT_TUPLE 440 -#define _JUMP_TO_TOP 441 +#define _ITER_CHECK_LIST 434 +#define _ITER_CHECK_RANGE 435 +#define _ITER_CHECK_TUPLE 436 +#define _ITER_JUMP_LIST 437 +#define _ITER_JUMP_RANGE 438 +#define _ITER_JUMP_TUPLE 439 +#define _ITER_NEXT_LIST 440 +#define _ITER_NEXT_LIST_TIER_TWO 441 +#define _ITER_NEXT_RANGE 442 +#define _ITER_NEXT_TUPLE 443 +#define _JUMP_TO_TOP 444 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 442 -#define _LOAD_ATTR_CLASS 443 +#define _LOAD_ATTR 445 +#define _LOAD_ATTR_CLASS 446 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 444 -#define _LOAD_ATTR_METHOD_LAZY_DICT 445 -#define _LOAD_ATTR_METHOD_NO_DICT 446 -#define _LOAD_ATTR_METHOD_WITH_VALUES 447 -#define _LOAD_ATTR_MODULE 448 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 449 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 450 -#define _LOAD_ATTR_PROPERTY_FRAME 451 -#define _LOAD_ATTR_SLOT 452 -#define _LOAD_ATTR_WITH_HINT 453 +#define _LOAD_ATTR_INSTANCE_VALUE 447 +#define _LOAD_ATTR_METHOD_LAZY_DICT 448 +#define _LOAD_ATTR_METHOD_NO_DICT 449 +#define _LOAD_ATTR_METHOD_WITH_VALUES 450 +#define _LOAD_ATTR_MODULE 451 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 452 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 453 +#define _LOAD_ATTR_PROPERTY_FRAME 454 +#define _LOAD_ATTR_SLOT 455 +#define _LOAD_ATTR_WITH_HINT 456 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 454 +#define _LOAD_BYTECODE 457 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 455 -#define _LOAD_CONST_INLINE_BORROW 456 -#define _LOAD_CONST_UNDER_INLINE 457 -#define _LOAD_CONST_UNDER_INLINE_BORROW 458 +#define _LOAD_CONST_INLINE 458 +#define _LOAD_CONST_INLINE_BORROW 459 +#define _LOAD_CONST_UNDER_INLINE 460 +#define _LOAD_CONST_UNDER_INLINE_BORROW 461 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 459 -#define _LOAD_FAST_0 460 -#define _LOAD_FAST_1 461 -#define _LOAD_FAST_2 462 -#define _LOAD_FAST_3 463 -#define _LOAD_FAST_4 464 -#define _LOAD_FAST_5 465 -#define _LOAD_FAST_6 466 -#define _LOAD_FAST_7 467 +#define _LOAD_FAST 462 +#define _LOAD_FAST_0 463 +#define _LOAD_FAST_1 464 +#define _LOAD_FAST_2 465 +#define _LOAD_FAST_3 466 +#define _LOAD_FAST_4 467 +#define _LOAD_FAST_5 468 +#define _LOAD_FAST_6 469 +#define _LOAD_FAST_7 470 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 468 -#define _LOAD_FAST_BORROW_0 469 -#define _LOAD_FAST_BORROW_1 470 -#define _LOAD_FAST_BORROW_2 471 -#define _LOAD_FAST_BORROW_3 472 -#define _LOAD_FAST_BORROW_4 473 -#define _LOAD_FAST_BORROW_5 474 -#define _LOAD_FAST_BORROW_6 475 -#define _LOAD_FAST_BORROW_7 476 +#define _LOAD_FAST_BORROW 471 +#define _LOAD_FAST_BORROW_0 472 +#define _LOAD_FAST_BORROW_1 473 +#define _LOAD_FAST_BORROW_2 474 +#define _LOAD_FAST_BORROW_3 475 +#define _LOAD_FAST_BORROW_4 476 +#define _LOAD_FAST_BORROW_5 477 +#define _LOAD_FAST_BORROW_6 478 +#define _LOAD_FAST_BORROW_7 479 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 477 -#define _LOAD_GLOBAL_BUILTINS 478 -#define _LOAD_GLOBAL_MODULE 479 +#define _LOAD_GLOBAL 480 +#define _LOAD_GLOBAL_BUILTINS 481 +#define _LOAD_GLOBAL_MODULE 482 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 480 -#define _LOAD_SMALL_INT_0 481 -#define _LOAD_SMALL_INT_1 482 -#define _LOAD_SMALL_INT_2 483 -#define _LOAD_SMALL_INT_3 484 -#define _LOAD_SPECIAL 485 +#define _LOAD_SMALL_INT 483 +#define _LOAD_SMALL_INT_0 484 +#define _LOAD_SMALL_INT_1 485 +#define _LOAD_SMALL_INT_2 486 +#define _LOAD_SMALL_INT_3 487 +#define _LOAD_SPECIAL 488 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 486 +#define _MAKE_CALLARGS_A_TUPLE 489 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 487 +#define _MAKE_WARM 490 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 488 -#define _MAYBE_EXPAND_METHOD_KW 489 -#define _MONITOR_CALL 490 -#define _MONITOR_CALL_KW 491 -#define _MONITOR_JUMP_BACKWARD 492 -#define _MONITOR_RESUME 493 +#define _MAYBE_EXPAND_METHOD 491 +#define _MAYBE_EXPAND_METHOD_KW 492 +#define _MONITOR_CALL 493 +#define _MONITOR_CALL_KW 494 +#define _MONITOR_JUMP_BACKWARD 495 +#define _MONITOR_RESUME 496 #define _NOP NOP -#define _POP_CALL 494 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 495 -#define _POP_CALL_ONE 496 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 497 -#define _POP_CALL_TWO 498 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 499 +#define _POP_CALL 497 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 498 +#define _POP_CALL_ONE 499 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 500 +#define _POP_CALL_TWO 501 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 502 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 500 -#define _POP_JUMP_IF_TRUE 501 +#define _POP_JUMP_IF_FALSE 503 +#define _POP_JUMP_IF_TRUE 504 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 502 -#define _POP_TOP_INT 503 -#define _POP_TOP_LOAD_CONST_INLINE 504 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 505 -#define _POP_TOP_NOP 506 -#define _POP_TOP_UNICODE 507 -#define _POP_TWO 508 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 509 +#define _POP_TOP_FLOAT 505 +#define _POP_TOP_INT 506 +#define _POP_TOP_LOAD_CONST_INLINE 507 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 508 +#define _POP_TOP_NOP 509 +#define _POP_TOP_UNICODE 510 +#define _POP_TWO 511 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 512 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 510 +#define _PUSH_FRAME 513 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 511 -#define _PY_FRAME_GENERAL 512 -#define _PY_FRAME_KW 513 -#define _QUICKEN_RESUME 514 -#define _REPLACE_WITH_TRUE 515 +#define _PUSH_NULL_CONDITIONAL 514 +#define _PY_FRAME_GENERAL 515 +#define _PY_FRAME_KW 516 +#define _QUICKEN_RESUME 517 +#define _REPLACE_WITH_TRUE 518 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 516 -#define _SEND 517 -#define _SEND_GEN_FRAME 518 +#define _SAVE_RETURN_OFFSET 519 +#define _SEND 520 +#define _SEND_GEN_FRAME 521 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 519 -#define _STORE_ATTR 520 -#define _STORE_ATTR_INSTANCE_VALUE 521 -#define _STORE_ATTR_SLOT 522 -#define _STORE_ATTR_WITH_HINT 523 +#define _START_EXECUTOR 522 +#define _STORE_ATTR 523 +#define _STORE_ATTR_INSTANCE_VALUE 524 +#define _STORE_ATTR_SLOT 525 +#define _STORE_ATTR_WITH_HINT 526 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 524 -#define _STORE_FAST_0 525 -#define _STORE_FAST_1 526 -#define _STORE_FAST_2 527 -#define _STORE_FAST_3 528 -#define _STORE_FAST_4 529 -#define _STORE_FAST_5 530 -#define _STORE_FAST_6 531 -#define _STORE_FAST_7 532 +#define _STORE_FAST 527 +#define _STORE_FAST_0 528 +#define _STORE_FAST_1 529 +#define _STORE_FAST_2 530 +#define _STORE_FAST_3 531 +#define _STORE_FAST_4 532 +#define _STORE_FAST_5 533 +#define _STORE_FAST_6 534 +#define _STORE_FAST_7 535 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 533 -#define _STORE_SUBSCR 534 -#define _STORE_SUBSCR_DICT 535 -#define _STORE_SUBSCR_LIST_INT 536 -#define _SWAP 537 -#define _SWAP_2 538 -#define _SWAP_3 539 -#define _TIER2_RESUME_CHECK 540 -#define _TO_BOOL 541 +#define _STORE_SLICE 536 +#define _STORE_SUBSCR 537 +#define _STORE_SUBSCR_DICT 538 +#define _STORE_SUBSCR_LIST_INT 539 +#define _SWAP 540 +#define _SWAP_2 541 +#define _SWAP_3 542 +#define _TIER2_RESUME_CHECK 543 +#define _TO_BOOL 544 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 542 +#define _TO_BOOL_LIST 545 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 543 +#define _TO_BOOL_STR 546 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 544 -#define _UNPACK_SEQUENCE_LIST 545 -#define _UNPACK_SEQUENCE_TUPLE 546 -#define _UNPACK_SEQUENCE_TWO_TUPLE 547 +#define _UNPACK_SEQUENCE 547 +#define _UNPACK_SEQUENCE_LIST 548 +#define _UNPACK_SEQUENCE_TUPLE 549 +#define _UNPACK_SEQUENCE_TWO_TUPLE 550 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 547 +#define MAX_UOP_ID 550 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index ff7e800aa9bb1a..bf361233560c55 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -24,7 +24,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RESUME_CHECK] = HAS_DEOPT_FLAG, - [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_FAST_CHECK] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG, [_LOAD_FAST_0] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_1] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_2] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, @@ -148,7 +148,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_LOCALS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_LOCALS] = HAS_ERROR_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL_CONDITIONAL] = HAS_ARG_FLAG, @@ -308,7 +308,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, - [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_JUMP_TO_TOP] = 0, [_SET_IP] = 0, @@ -330,12 +330,14 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_CONST_UNDER_INLINE] = 0, [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, [_CHECK_FUNCTION] = HAS_DEOPT_FLAG, - [_START_EXECUTOR] = 0, + [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, [_DEOPT] = 0, + [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, [_ERROR_POP_N] = HAS_ARG_FLAG, - [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, + [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, + [_COLD_EXIT] = HAS_ESCAPES_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { @@ -419,6 +421,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", @@ -499,6 +502,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", + [_HANDLE_PENDING_AND_DEOPT] = "_HANDLE_PENDING_AND_DEOPT", [_IMPORT_FROM] = "_IMPORT_FROM", [_IMPORT_NAME] = "_IMPORT_NAME", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", @@ -1297,10 +1301,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _DEOPT: return 0; + case _HANDLE_PENDING_AND_DEOPT: + return 0; case _ERROR_POP_N: return 0; case _TIER2_RESUME_CHECK: return 0; + case _COLD_EXIT: + return 0; default: return -1; } diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 9c35684c945b3e..a7d872f3ec4392 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -329,15 +329,16 @@ Once the GC knows the list of unreachable objects, a very delicate process start with the objective of completely destroying these objects. Roughly, the process follows these steps in order: -1. Handle and clear weak references (if any). Weak references to unreachable objects - are set to `None`. If the weak reference has an associated callback, the callback - is enqueued to be called once the clearing of weak references is finished. We only - invoke callbacks for weak references that are themselves reachable. If both the weak - reference and the pointed-to object are unreachable we do not execute the callback. - This is partly for historical reasons: the callback could resurrect an unreachable - object and support for weak references predates support for object resurrection. - Ignoring the weak reference's callback is fine because both the object and the weakref - are going away, so it's legitimate to say the weak reference is going away first. +1. Handle weak references with callbacks (if any). If the weak reference has + an associated callback, the callback is enqueued to be called after the weak + reference is cleared. We only invoke callbacks for weak references that + are themselves reachable. If both the weak reference and the pointed-to + object are unreachable we do not execute the callback. This is partly for + historical reasons: the callback could resurrect an unreachable object + and support for weak references predates support for object resurrection. + Ignoring the weak reference's callback is fine because both the object and + the weakref are going away, so it's legitimate to say the weak reference is + going away first. 2. If an object has legacy finalizers (`tp_del` slot) move it to the `gc.garbage` list. 3. Call the finalizers (`tp_finalize` slot) and mark the objects as already @@ -346,7 +347,12 @@ follows these steps in order: 4. Deal with resurrected objects. If some objects have been resurrected, the GC finds the new subset of objects that are still unreachable by running the cycle detection algorithm again and continues with them. -5. Call the `tp_clear` slot of every object so all internal links are broken and +5. Clear any weak references that still refer to unreachable objects. The + `wr_object` attribute for these weakrefs are set to `None`. Note that some + of these weak references maybe have been newly created during the running of + finalizers in step 3. Also, clear any weak references that are part of the + unreachable set. +6. Call the `tp_clear` slot of every object so all internal links are broken and the reference counts fall to 0, triggering the destruction of all unreachable objects. diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 4a310a402358b6..325efed274aed7 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -172,7 +172,18 @@ class Argparse(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) +class Difflib(ThemeSection): + """A 'git diff'-like theme for `difflib.unified_diff`.""" + added: str = ANSIColors.GREEN + context: str = ANSIColors.RESET # context lines + header: str = ANSIColors.BOLD # eg "---" and "+++" lines + hunk: str = ANSIColors.CYAN # the "@@" lines + removed: str = ANSIColors.RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA keyword: str = ANSIColors.BOLD_BLUE @@ -186,7 +197,7 @@ class Syntax(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Traceback(ThemeSection): type: str = ANSIColors.BOLD_MAGENTA message: str = ANSIColors.MAGENTA @@ -198,7 +209,7 @@ class Traceback(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Unittest(ThemeSection): passed: str = ANSIColors.GREEN warn: str = ANSIColors.YELLOW @@ -207,7 +218,7 @@ class Unittest(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Theme: """A suite of themes for all sections of Python. @@ -215,6 +226,7 @@ class Theme: below. """ argparse: Argparse = field(default_factory=Argparse) + difflib: Difflib = field(default_factory=Difflib) syntax: Syntax = field(default_factory=Syntax) traceback: Traceback = field(default_factory=Traceback) unittest: Unittest = field(default_factory=Unittest) @@ -223,6 +235,7 @@ def copy_with( self, *, argparse: Argparse | None = None, + difflib: Difflib | None = None, syntax: Syntax | None = None, traceback: Traceback | None = None, unittest: Unittest | None = None, @@ -234,6 +247,7 @@ def copy_with( """ return type(self)( argparse=argparse or self.argparse, + difflib=difflib or self.difflib, syntax=syntax or self.syntax, traceback=traceback or self.traceback, unittest=unittest or self.unittest, @@ -249,6 +263,7 @@ def no_colors(cls) -> Self: """ return cls( argparse=Argparse.no_colors(), + difflib=Difflib.no_colors(), syntax=Syntax.no_colors(), traceback=Traceback.no_colors(), unittest=Unittest.no_colors(), diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index bc35823f70144e..0955005df5ccee 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -1083,7 +1083,7 @@ def fromisocalendar(cls, year, week, day): @classmethod def strptime(cls, date_string, format): - """Parse a date string according to the given format (like time.strptime()).""" + """Parse string according to the given date format (like time.strptime()).""" import _strptime return _strptime._strptime_datetime_date(cls, date_string, format) @@ -1310,7 +1310,7 @@ def __reduce__(self): class tzinfo: - """Abstract base class for time zone info classes. + """Abstract base class for time zone info objects. Subclasses must override the tzname(), utcoffset() and dst() methods. """ @@ -1468,7 +1468,7 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold @classmethod def strptime(cls, date_string, format): - """string, format -> new time parsed from a string (like time.strptime()).""" + """Parse string according to the given time format (like time.strptime()).""" import _strptime return _strptime._strptime_datetime_time(cls, date_string, format) @@ -1776,7 +1776,7 @@ def __reduce__(self): class datetime(date): - """datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]]) + """A combination of a date and a time. The year, month and day arguments are required. tzinfo may be None, or an instance of a tzinfo subclass. The remaining arguments may be ints. @@ -2209,7 +2209,7 @@ def __str__(self): @classmethod def strptime(cls, date_string, format): - 'string, format -> new datetime parsed from a string (like time.strptime()).' + """Parse string according to the given date and time format (like time.strptime()).""" import _strptime return _strptime._strptime_datetime_datetime(cls, date_string, format) @@ -2435,6 +2435,8 @@ def _isoweek1monday(year): class timezone(tzinfo): + """Fixed offset from UTC implementation of tzinfo.""" + __slots__ = '_offset', '_name' # Sentinel value to disallow None diff --git a/Lib/ast.py b/Lib/ast.py index 6d3daf64f5c6d7..983ac1710d0205 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -57,53 +57,60 @@ def literal_eval(node_or_string): Caution: A complex expression can overflow the C stack and cause a crash. """ if isinstance(node_or_string, str): - node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval') - if isinstance(node_or_string, Expression): + node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval').body + elif isinstance(node_or_string, Expression): node_or_string = node_or_string.body - def _raise_malformed_node(node): - msg = "malformed node or string" - if lno := getattr(node, 'lineno', None): - msg += f' on line {lno}' - raise ValueError(msg + f': {node!r}') - def _convert_num(node): - if not isinstance(node, Constant) or type(node.value) not in (int, float, complex): - _raise_malformed_node(node) + return _convert_literal(node_or_string) + + +def _convert_literal(node): + """ + Used by `literal_eval` to convert an AST node into a value. + """ + if isinstance(node, Constant): return node.value - def _convert_signed_num(node): - if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): - operand = _convert_num(node.operand) - if isinstance(node.op, UAdd): - return + operand - else: - return - operand - return _convert_num(node) - def _convert(node): - if isinstance(node, Constant): - return node.value - elif isinstance(node, Tuple): - return tuple(map(_convert, node.elts)) - elif isinstance(node, List): - return list(map(_convert, node.elts)) - elif isinstance(node, Set): - return set(map(_convert, node.elts)) - elif (isinstance(node, Call) and isinstance(node.func, Name) and - node.func.id == 'set' and node.args == node.keywords == []): - return set() - elif isinstance(node, Dict): - if len(node.keys) != len(node.values): - _raise_malformed_node(node) - return dict(zip(map(_convert, node.keys), - map(_convert, node.values))) - elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): - left = _convert_signed_num(node.left) - right = _convert_num(node.right) - if isinstance(left, (int, float)) and isinstance(right, complex): - if isinstance(node.op, Add): - return left + right - else: - return left - right - return _convert_signed_num(node) - return _convert(node_or_string) + if isinstance(node, Dict) and len(node.keys) == len(node.values): + return dict(zip( + map(_convert_literal, node.keys), + map(_convert_literal, node.values), + )) + if isinstance(node, Tuple): + return tuple(map(_convert_literal, node.elts)) + if isinstance(node, List): + return list(map(_convert_literal, node.elts)) + if isinstance(node, Set): + return set(map(_convert_literal, node.elts)) + if ( + isinstance(node, Call) and isinstance(node.func, Name) + and node.func.id == 'set' and node.args == node.keywords == [] + ): + return set() + if ( + isinstance(node, UnaryOp) + and isinstance(node.op, (UAdd, USub)) + and isinstance(node.operand, Constant) + and type(operand := node.operand.value) in (int, float, complex) + ): + if isinstance(node.op, UAdd): + return + operand + else: + return - operand + if ( + isinstance(node, BinOp) + and isinstance(node.op, (Add, Sub)) + and isinstance(node.left, (Constant, UnaryOp)) + and isinstance(node.right, Constant) + and type(left := _convert_literal(node.left)) in (int, float) + and type(right := _convert_literal(node.right)) is complex + ): + if isinstance(node.op, Add): + return left + right + else: + return left - right + msg = "malformed node or string" + if lno := getattr(node, 'lineno', None): + msg += f' on line {lno}' + raise ValueError(msg + f': {node!r}') def dump( diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index ff3a69d1e17297..5ef2e1f9efc9ed 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,7 +1,6 @@ import argparse import ast import asyncio -import asyncio.tools import concurrent.futures import contextvars import inspect @@ -11,6 +10,9 @@ import threading import types import warnings +from asyncio.tools import (TaskTableOutputFormat, + display_awaited_by_tasks_table, + display_awaited_by_tasks_tree) from _colorize import get_theme from _pyrepl.console import InteractiveColoredConsole @@ -153,6 +155,11 @@ def interrupt(self) -> None: "ps", help="Display a table of all pending tasks in a process" ) ps.add_argument("pid", type=int, help="Process ID to inspect") + formats = [fmt.value for fmt in TaskTableOutputFormat] + formats_to_show = [fmt for fmt in formats + if fmt != TaskTableOutputFormat.bsv.value] + ps.add_argument("--format", choices=formats, default="table", + metavar=f"{{{','.join(formats_to_show)}}}") pstree = subparsers.add_parser( "pstree", help="Display a tree of all pending tasks in a process" ) @@ -160,10 +167,10 @@ def interrupt(self) -> None: args = parser.parse_args() match args.command: case "ps": - asyncio.tools.display_awaited_by_tasks_table(args.pid) + display_awaited_by_tasks_table(args.pid, format=args.format) sys.exit(0) case "pstree": - asyncio.tools.display_awaited_by_tasks_tree(args.pid) + display_awaited_by_tasks_tree(args.pid) sys.exit(0) case None: pass # continue to the interactive shell diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 520d4b398545bf..8cbb71f708537f 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -636,7 +636,7 @@ def _check_running(self): def _run_forever_setup(self): """Prepare the run loop to process events. - This method exists so that custom custom event loop subclasses (e.g., event loops + This method exists so that custom event loop subclasses (e.g., event loops that integrate a GUI event loop with Python's event loop) have access to all the loop setup logic. """ @@ -656,7 +656,7 @@ def _run_forever_setup(self): def _run_forever_cleanup(self): """Clean up after an event loop finishes the looping over events. - This method exists so that custom custom event loop subclasses (e.g., event loops + This method exists so that custom event loop subclasses (e.g., event loops that integrate a GUI event loop with Python's event loop) have access to all the loop cleanup logic. """ diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 2ad65d8648e6c5..845aed4c6a3b35 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -62,7 +62,6 @@ async def staggered_race(coro_fns, delay, *, loop=None): coroutine's entry is ``None``. """ - # TODO: when we have aiter() and anext(), allow async iterables in coro_fns. loop = loop or events.get_running_loop() parent_task = tasks.current_task(loop) enum_coro_fns = enumerate(coro_fns) diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 64aac4cc50d15a..c8c01f36474183 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -271,7 +271,6 @@ def connection_lost(self, exc): self._closed.set_exception(exc) super().connection_lost(exc) self._stream_reader_wr = None - self._stream_writer = None self._task = None self._transport = None diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 2683f34cc7113b..3887fb8ab5bf52 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,8 +1,9 @@ """Tools to analyze tasks running in asyncio programs.""" -from collections import defaultdict, namedtuple +from collections import defaultdict +import csv from itertools import count -from enum import Enum +from enum import Enum, StrEnum, auto import sys from _remote_debugging import RemoteUnwinder, FrameInfo @@ -232,18 +233,56 @@ def _get_awaited_by_tasks(pid: int) -> list: sys.exit(1) -def display_awaited_by_tasks_table(pid: int) -> None: +class TaskTableOutputFormat(StrEnum): + table = auto() + csv = auto() + bsv = auto() + # 🍌SV is not just a format. It's a lifestyle. A philosophy. + # https://www.youtube.com/watch?v=RrsVi1P6n0w + + +def display_awaited_by_tasks_table(pid, *, format=TaskTableOutputFormat.table): """Build and print a table of all pending tasks under `pid`.""" tasks = _get_awaited_by_tasks(pid) table = build_task_table(tasks) - # Print the table in a simple tabular format - print( - f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine stack':<50} {'awaiter chain':<50} {'awaiter name':<15} {'awaiter id':<15}" - ) - print("-" * 180) + format = TaskTableOutputFormat(format) + if format == TaskTableOutputFormat.table: + _display_awaited_by_tasks_table(table) + else: + _display_awaited_by_tasks_csv(table, format=format) + + +_row_header = ('tid', 'task id', 'task name', 'coroutine stack', + 'awaiter chain', 'awaiter name', 'awaiter id') + + +def _display_awaited_by_tasks_table(table): + """Print the table in a simple tabular format.""" + print(_fmt_table_row(*_row_header)) + print('-' * 180) for row in table: - print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") + print(_fmt_table_row(*row)) + + +def _fmt_table_row(tid, task_id, task_name, coro_stack, + awaiter_chain, awaiter_name, awaiter_id): + # Format a single row for the table format + return (f'{tid:<10} {task_id:<20} {task_name:<20} {coro_stack:<50} ' + f'{awaiter_chain:<50} {awaiter_name:<15} {awaiter_id:<15}') + + +def _display_awaited_by_tasks_csv(table, *, format): + """Print the table in CSV format""" + if format == TaskTableOutputFormat.csv: + delimiter = ',' + elif format == TaskTableOutputFormat.bsv: + delimiter = '\N{BANANA}' + else: + raise ValueError(f"Unknown output format: {format}") + csv_writer = csv.writer(sys.stdout, delimiter=delimiter) + csv_writer.writerow(_row_header) + csv_writer.writerows(table) def display_awaited_by_tasks_tree(pid: int) -> None: diff --git a/Lib/calendar.py b/Lib/calendar.py index 3be1b50500eb07..45bb265a65602c 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -14,8 +14,9 @@ __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", "monthcalendar", "prmonth", "month", "prcal", "calendar", - "timegm", "month_name", "month_abbr", "day_name", "day_abbr", - "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", + "timegm", "month_name", "month_abbr", "standalone_month_name", + "standalone_month_abbr", "day_name", "day_abbr", "Calendar", + "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", "LocaleHTMLCalendar", "weekheader", "Day", "Month", "JANUARY", "FEBRUARY", "MARCH", "APRIL", "MAY", "JUNE", "JULY", @@ -139,6 +140,16 @@ def __len__(self): month_name = _localized_month('%B') month_abbr = _localized_month('%b') +# On platforms that support the %OB and %Ob specifiers, they are used +# to get the standalone form of the month name. This is required for +# some languages such as Greek, Slavic, and Baltic languages. +try: + standalone_month_name = _localized_month('%OB') + standalone_month_abbr = _localized_month('%Ob') +except ValueError: + standalone_month_name = month_name + standalone_month_abbr = month_abbr + def isleap(year): """Return True for leap years, False for non-leap years.""" @@ -377,7 +388,7 @@ def formatmonthname(self, theyear, themonth, width, withyear=True): """ _validate_month(themonth) - s = month_name[themonth] + s = standalone_month_name[themonth] if withyear: s = "%s %r" % (s, theyear) return s.center(width) @@ -510,9 +521,9 @@ def formatmonthname(self, theyear, themonth, withyear=True): """ _validate_month(themonth) if withyear: - s = '%s %s' % (month_name[themonth], theyear) + s = '%s %s' % (standalone_month_name[themonth], theyear) else: - s = '%s' % month_name[themonth] + s = standalone_month_name[themonth] return '%s' % ( self.cssclass_month_head, s) diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index e717222cf98b32..d6ac4b3e0b675f 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -44,7 +44,7 @@ def __dir__(): - return __all__ + ('__author__', '__doc__') + return __all__ + ['__author__', '__doc__'] def __getattr__(name): diff --git a/Lib/difflib.py b/Lib/difflib.py index 487936dbf47cdc..fedc85009aa03b 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -30,6 +30,7 @@ 'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff', 'unified_diff', 'diff_bytes', 'HtmlDiff', 'Match'] +from _colorize import can_colorize, get_theme from heapq import nlargest as _nlargest from collections import namedtuple as _namedtuple from types import GenericAlias @@ -1094,7 +1095,7 @@ def _format_range_unified(start, stop): return '{},{}'.format(beginning, length) def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', - tofiledate='', n=3, lineterm='\n'): + tofiledate='', n=3, lineterm='\n', *, color=False): r""" Compare two sequences of lines; generate the delta as a unified diff. @@ -1111,6 +1112,10 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', For inputs that do not have trailing newlines, set the lineterm argument to "" so that the output will be uniformly newline free. + Set 'color' to True to enable output in color, similar to + 'git diff --color'. Even if enabled, it can be + controlled using environment variables such as 'NO_COLOR'. + The unidiff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. @@ -1134,6 +1139,11 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', four """ + if color and can_colorize(): + t = get_theme(force_color=True).difflib + else: + t = get_theme(force_no_color=True).difflib + _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): @@ -1141,25 +1151,25 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', started = True fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' todate = '\t{}'.format(tofiledate) if tofiledate else '' - yield '--- {}{}{}'.format(fromfile, fromdate, lineterm) - yield '+++ {}{}{}'.format(tofile, todate, lineterm) + yield f'{t.header}--- {fromfile}{fromdate}{lineterm}{t.reset}' + yield f'{t.header}+++ {tofile}{todate}{lineterm}{t.reset}' first, last = group[0], group[-1] file1_range = _format_range_unified(first[1], last[2]) file2_range = _format_range_unified(first[3], last[4]) - yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm) + yield f'{t.hunk}@@ -{file1_range} +{file2_range} @@{lineterm}{t.reset}' for tag, i1, i2, j1, j2 in group: if tag == 'equal': for line in a[i1:i2]: - yield ' ' + line + yield f'{t.context} {line}{t.reset}' continue if tag in {'replace', 'delete'}: for line in a[i1:i2]: - yield '-' + line + yield f'{t.removed}-{line}{t.reset}' if tag in {'replace', 'insert'}: for line in b[j1:j2]: - yield '+' + line + yield f'{t.added}+{line}{t.reset}' ######################################################################## diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index aa641e94a8b336..4bd85990e8614a 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "25.1.1" +_PIP_VERSION = "25.2" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora diff --git a/Lib/ensurepip/_bundled/pip-25.1.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl similarity index 58% rename from Lib/ensurepip/_bundled/pip-25.1.1-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl index 2fdcfbf9ff8139..e14bb3f37c0ff4 100644 Binary files a/Lib/ensurepip/_bundled/pip-25.1.1-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl differ diff --git a/Lib/glob.py b/Lib/glob.py index 1e48fe43167200..5d42077003a240 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -122,21 +122,6 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): return [basename] return [] -_deprecated_function_message = ( - "{name} is deprecated and will be removed in Python {remove}. Use " - "glob.glob and pass a directory to its root_dir argument instead." -) - -def glob0(dirname, pattern): - import warnings - warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) - return _glob0(dirname, pattern, None, False) - -def glob1(dirname, pattern): - import warnings - warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) - return _glob1(dirname, pattern, None, False) - # This helper function recursively yields relative pathnames inside a literal # directory. diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 694b1b09a0567c..74349bb63d66e2 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -426,7 +426,7 @@ def OutputString(self, attrs=None): ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P # Start of group 'val' - "(?:[^\\"]|\\.)*" # Any double-quoted string + "(?:\\"|.)*?" # Any double-quoted string | # or # Special case for "expires" attr (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day diff --git a/Lib/numbers.py b/Lib/numbers.py index a2913e32cfada7..37fddb8917727b 100644 --- a/Lib/numbers.py +++ b/Lib/numbers.py @@ -290,18 +290,27 @@ def conjugate(self): class Rational(Real): - """.numerator and .denominator should be in lowest terms.""" + """To Real, Rational adds numerator and denominator properties. + + The numerator and denominator values should be in lowest terms, + with a positive denominator. + """ __slots__ = () @property @abstractmethod def numerator(self): + """The numerator of a rational number in lowest terms.""" raise NotImplementedError @property @abstractmethod def denominator(self): + """The denominator of a rational number in lowest terms. + + This denominator should be positive. + """ raise NotImplementedError # Concrete implementation of Real's conversion to float. diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 49e0986517ce97..c583b74ce54fc6 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -412,7 +412,13 @@ def _init_non_posix(vars): vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) - vars['TZPATH'] = '' + # No standard path exists on Windows for this, but we'll check + # whether someone is imitating a POSIX-like layout + check_tzpath = os.path.join(vars['prefix'], 'share', 'zoneinfo') + if os.path.exists(check_tzpath): + vars['TZPATH'] = check_tzpath + else: + vars['TZPATH'] = '' # # public APIs diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 45f58eb8ac93cf..c603ba019ab481 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -353,7 +353,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, fileobj = _StreamProxy(fileobj) comptype = fileobj.getcomptype() - self.name = name or "" + self.name = os.fspath(name) if name is not None else "" self.mode = mode self.comptype = comptype self.fileobj = fileobj @@ -2723,6 +2723,9 @@ def makelink_with_filter(self, tarinfo, targetpath, return else: if os.path.exists(tarinfo._link_target): + if os.path.lexists(targetpath): + # Avoid FileExistsError on following os.link. + os.unlink(targetpath) os.link(tarinfo._link_target, targetpath) return except symlink_exception: diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py index 0ce42276bfe3d6..4a050792df73c4 100644 --- a/Lib/test/_test_eintr.py +++ b/Lib/test/_test_eintr.py @@ -380,6 +380,8 @@ def os_open(self, path): @unittest.skipIf(sys.platform == "darwin", "hangs under macOS; see bpo-25234, bpo-35363") + @unittest.skipIf(sys.platform.startswith('netbsd'), + "hangs on NetBSD; see gh-137397") def test_os_open(self): self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", self.os_open) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 3bd3a866570042..4e42ae52765d0f 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2147,14 +2147,20 @@ def test_fromisocalendar_value_errors(self): (10000, 1, 1), (0, 1, 1), (9999999, 1, 1), + ] + for isocal in isocals: + with self.subTest(isocal=isocal): + with self.assertRaises(ValueError): + self.theclass.fromisocalendar(*isocal) + + isocals = [ (2<<32, 1, 1), (2019, 2<<32, 1), (2019, 1, 2<<32), ] - for isocal in isocals: with self.subTest(isocal=isocal): - with self.assertRaises(ValueError): + with self.assertRaises((ValueError, OverflowError)): self.theclass.fromisocalendar(*isocal) def test_fromisocalendar_type_errors(self): @@ -2301,7 +2307,7 @@ def test_isoformat_timezone(self): dt = dt_base.replace(tzinfo=tzi) exp = exp_base + exp_tz with self.subTest(tzi=tzi): - assert dt.isoformat() == exp + self.assertEqual(dt.isoformat(), exp) def test_format(self): dt = self.theclass(2007, 9, 10, 4, 5, 1, 123) @@ -3349,7 +3355,7 @@ def test_fromisoformat_timezone(self): with self.subTest(tstr=dtstr): dt_rt = self.theclass.fromisoformat(dtstr) - assert dt == dt_rt, dt_rt + self.assertEqual(dt_rt, dt) def test_fromisoformat_separators(self): separators = [ @@ -3865,7 +3871,7 @@ def test_isoformat_timezone(self): t = t_base.replace(tzinfo=tzi) exp = exp_base + exp_tz with self.subTest(tzi=tzi): - assert t.isoformat() == exp + self.assertEqual(t.isoformat(), exp) def test_1653736(self): # verify it doesn't accept extra keyword arguments @@ -4350,7 +4356,7 @@ def utcoffset(self, t): elif x is d2: expected = -1 else: - assert y is d2 + self.assertIs(y, d2) expected = 1 self.assertEqual(got, expected) @@ -4678,7 +4684,7 @@ def test_fromisoformat_timezone(self): with self.subTest(tstr=tstr): t_rt = self.theclass.fromisoformat(tstr) - assert t == t_rt + self.assertEqual(t_rt, t) def test_fromisoformat_timespecs(self): time_bases = [ @@ -5515,7 +5521,7 @@ def utcoffset(self, t): elif x is d2: expected = timedelta(minutes=(11-59)-0) else: - assert y is d2 + self.assertIs(y, d2) expected = timedelta(minutes=0-(11-59)) self.assertEqual(got, expected) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 72b8ea89e62ee0..d94fb84a743828 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -536,7 +536,7 @@ def normalize_test_name(test_full_name: str, *, if is_error and short_name in _TEST_LIFECYCLE_HOOKS: if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): # if setUpModule() or tearDownModule() failed, don't filter - # tests with the test file name, don't use use filters. + # tests with the test file name, don't use filters. return None # This means that we have a failure in a life-cycle hook, diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 100438bf71d3a6..4bfd01ed14a0a1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -541,10 +541,14 @@ def has_no_debug_ranges(): except ImportError: raise unittest.SkipTest("_testinternalcapi required") return not _testcapi.config_get('code_debug_ranges') - return not bool(config['code_debug_ranges']) def requires_debug_ranges(reason='requires co_positions / debug_ranges'): - return unittest.skipIf(has_no_debug_ranges(), reason) + try: + skip = has_no_debug_ranges() + except unittest.SkipTest as e: + skip = True + reason = e.args[0] if e.args else reason + return unittest.skipIf(skip, reason) MS_WINDOWS = (sys.platform == 'win32') @@ -1681,7 +1685,7 @@ def check__all__(test_case, module, name_of_module=None, extra=(), 'module'. The 'name_of_module' argument can specify (as a string or tuple thereof) - what module(s) an API could be defined in in order to be detected as a + what module(s) an API could be defined in order to be detected as a public API. One case for this is when 'module' imports part of its public API from other modules, possibly a C backend (like 'csv' and its '_csv'). diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index f35b1194308262..8ded9f99248372 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -72,6 +72,8 @@ def check_all(self, modname): all_set = set(all_list) self.assertCountEqual(all_set, all_list, "in module {}".format(modname)) self.assertEqual(keys, all_set, "in module {}".format(modname)) + # Verify __dir__ is non-empty and doesn't produce an error + self.assertTrue(dir(sys.modules[modname])) def walk_modules(self, basedir, modpath): for fn in sorted(os.listdir(basedir)): diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 13dcb5238945b6..1e6f60074308e2 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -220,6 +220,131 @@ def test_negative_locations_for_compile(self): # This also must not crash: ast.parse(tree, optimize=2) + def test_docstring_optimization_single_node(self): + # https://github.com/python/cpython/issues/137308 + class_example1 = textwrap.dedent(''' + class A: + """Docstring""" + ''') + class_example2 = textwrap.dedent(''' + class A: + """ + Docstring""" + ''') + def_example1 = textwrap.dedent(''' + def some(): + """Docstring""" + ''') + def_example2 = textwrap.dedent(''' + def some(): + """Docstring + """ + ''') + async_def_example1 = textwrap.dedent(''' + async def some(): + """Docstring""" + ''') + async_def_example2 = textwrap.dedent(''' + async def some(): + """ + Docstring + """ + ''') + for code in [ + class_example1, + class_example2, + def_example1, + def_example2, + async_def_example1, + async_def_example2, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + self.assertEqual(len(mod.body[0].body), 1) + if opt_level == 2: + pass_stmt = mod.body[0].body[0] + self.assertIsInstance(pass_stmt, ast.Pass) + self.assertEqual( + vars(pass_stmt), + { + 'lineno': 3, + 'col_offset': 4, + 'end_lineno': 3, + 'end_col_offset': 8, + }, + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + + def test_docstring_optimization_multiple_nodes(self): + # https://github.com/python/cpython/issues/137308 + class_example = textwrap.dedent( + """ + class A: + ''' + Docstring + ''' + x = 1 + """ + ) + + def_example = textwrap.dedent( + """ + def some(): + ''' + Docstring + + ''' + x = 1 + """ + ) + + async_def_example = textwrap.dedent( + """ + async def some(): + + '''Docstring + + ''' + x = 1 + """ + ) + + for code in [ + class_example, + def_example, + async_def_example, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + if opt_level == 2: + self.assertNotIsInstance( + mod.body[0].body[0], + (ast.Pass, ast.Expr), + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice self.assertIsNone(slc.upper) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index bc39c86b8cf62d..589a21baf7bd61 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -8,6 +8,7 @@ import io import locale import os +import platform import sys import time @@ -546,7 +547,8 @@ def test_days(self): self.assertEqual(value[::-1], list(reversed(value))) def test_months(self): - for attr in "month_name", "month_abbr": + for attr in ("month_name", "month_abbr", "standalone_month_name", + "standalone_month_abbr"): value = getattr(calendar, attr) self.assertEqual(len(value), 13) self.assertEqual(len(value[:]), 13) @@ -556,6 +558,38 @@ def test_months(self): # verify it "acts like a sequence" in two forms of iteration self.assertEqual(value[::-1], list(reversed(value))) + @support.run_with_locale('LC_ALL', 'pl_PL') + @unittest.skipUnless(sys.platform == 'darwin' or platform.libc_ver()[0] == 'glibc', + "Guaranteed to work with glibc and macOS") + def test_standalone_month_name_and_abbr_pl_locale(self): + expected_standalone_month_names = [ + "", "styczeń", "luty", "marzec", "kwiecień", "maj", "czerwiec", + "lipiec", "sierpień", "wrzesień", "październik", "listopad", + "grudzień" + ] + expected_standalone_month_abbr = [ + "", "sty", "lut", "mar", "kwi", "maj", "cze", + "lip", "sie", "wrz", "paź", "lis", "gru" + ] + self.assertEqual( + list(calendar.standalone_month_name), + expected_standalone_month_names + ) + self.assertEqual( + list(calendar.standalone_month_abbr), + expected_standalone_month_abbr + ) + + def test_standalone_month_name_and_abbr_C_locale(self): + # Ensure that the standalone month names and abbreviations are + # equal to the regular month names and abbreviations for + # the "C" locale. + with calendar.different_locale("C"): + self.assertListEqual(list(calendar.month_name), + list(calendar.standalone_month_name)) + self.assertListEqual(list(calendar.month_abbr), + list(calendar.standalone_month_abbr)) + def test_locale_text_calendar(self): try: cal = calendar.LocaleTextCalendar(locale='') diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index ade55338e63b69..4967f02b007e06 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -6,7 +6,7 @@ import textwrap from test import support -from test.support import import_helper +from test.support import import_helper, force_not_colorized from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE from test.support.script_helper import assert_python_failure, assert_python_ok from test.support.testcase import ExceptionIsLikeMixin @@ -337,6 +337,10 @@ def test_err_writeunraisable(self): self.assertIsNone(cm.unraisable.err_msg) self.assertIsNone(cm.unraisable.object) + @force_not_colorized + def test_err_writeunraisable_lines(self): + writeunraisable = _testcapi.err_writeunraisable + with (support.swap_attr(sys, 'unraisablehook', None), support.captured_stderr() as stderr): writeunraisable(CustomError('oops!'), hex) @@ -387,6 +391,10 @@ def test_err_formatunraisable(self): self.assertIsNone(cm.unraisable.err_msg) self.assertIsNone(cm.unraisable.object) + @force_not_colorized + def test_err_formatunraisable_lines(self): + formatunraisable = _testcapi.err_formatunraisable + with (support.swap_attr(sys, 'unraisablehook', None), support.captured_stderr() as stderr): formatunraisable(CustomError('oops!'), b'Error in %R', []) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index f30a1874ab96d4..3ed7a360d64e3c 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -489,6 +489,7 @@ def test_unmached_quote(self): self.assertRegex(err.decode('ascii', 'ignore'), 'SyntaxError') self.assertEqual(b'', out) + @force_not_colorized def test_stdout_flush_at_shutdown(self): # Issue #5319: if stdout.flush() fails at shutdown, an error should # be printed out. diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index d8666f7290e72e..fd7769e8c275d3 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3293,7 +3293,7 @@ def test_code_page_name(self): codecs.code_page_encode, 932, '\xff') self.assertRaisesRegex(UnicodeDecodeError, 'cp932', codecs.code_page_decode, 932, b'\x81\x00', 'strict', True) - self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8', + self.assertRaisesRegex(UnicodeDecodeError, 'cp65001', codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index d9d61e5c2053e3..f33e4b3256a9b9 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -736,7 +736,7 @@ def validate_abstract_methods(self, abc, *names): stubs = methodstubs.copy() del stubs[name] C = type('C', (abc,), stubs) - self.assertRaises(TypeError, C, name) + self.assertRaises(TypeError, C) def validate_isinstance(self, abc, name): stub = lambda s, *args: 0 @@ -963,7 +963,7 @@ class AnextOnly: async def __anext__(self): raise StopAsyncIteration self.assertNotIsInstance(AnextOnly(), AsyncIterator) - self.validate_abstract_methods(AsyncIterator, '__anext__', '__aiter__') + self.validate_abstract_methods(AsyncIterator, '__anext__') def test_Iterable(self): # Check some non-iterables @@ -1159,7 +1159,7 @@ def test_Iterator(self): for x in samples: self.assertIsInstance(x, Iterator) self.assertIsSubclass(type(x), Iterator) - self.validate_abstract_methods(Iterator, '__next__', '__iter__') + self.validate_abstract_methods(Iterator, '__next__') # Issue 10565 class NextOnly: @@ -1843,8 +1843,7 @@ def test_Mapping(self): for sample in [dict]: self.assertIsInstance(sample(), Mapping) self.assertIsSubclass(sample, Mapping) - self.validate_abstract_methods(Mapping, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Mapping, '__iter__', '__len__', '__getitem__') class MyMapping(Mapping): def __len__(self): return 0 @@ -1859,7 +1858,7 @@ def test_MutableMapping(self): for sample in [dict]: self.assertIsInstance(sample(), MutableMapping) self.assertIsSubclass(sample, MutableMapping) - self.validate_abstract_methods(MutableMapping, '__contains__', '__iter__', '__len__', + self.validate_abstract_methods(MutableMapping, '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__') def test_MutableMapping_subclass(self): @@ -1898,8 +1897,7 @@ def test_Sequence(self): self.assertIsInstance(memoryview(b""), Sequence) self.assertIsSubclass(memoryview, Sequence) self.assertIsSubclass(str, Sequence) - self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Sequence, '__len__', '__getitem__') def test_Sequence_mixins(self): class SequenceSubclass(Sequence): @@ -1954,8 +1952,8 @@ def test_MutableSequence(self): self.assertIsSubclass(sample, MutableSequence) self.assertIsSubclass(array.array, MutableSequence) self.assertNotIsSubclass(str, MutableSequence) - self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', - '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') + self.validate_abstract_methods(MutableSequence, '__len__', '__getitem__', + '__setitem__', '__delitem__', 'insert') def test_MutableSequence_mixins(self): # Test the mixins of MutableSequence by creating a minimal concrete diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 57e5f29b015637..8a66be9b331262 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1723,6 +1723,21 @@ def test_compound(self): self.assertIs(res, v[3]) self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) + def test_exception(self): + # See gh-137288 + class Foo: + def __bool__(self): + raise NotImplementedError() + + a = Foo() + b = Foo() + + with self.assertRaises(NotImplementedError): + bool(a) + + with self.assertRaises(NotImplementedError): + c = a or b + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 99b315b47e2530..43812248104c91 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -49,6 +49,7 @@ def test_interpreter_shutdown(self): self.assertFalse(err) self.assertEqual(out.strip(), b"apple") + @support.force_not_colorized def test_submit_after_interpreter_shutdown(self): # Test the atexit hook for shutdown of worker threads and processes rc, out, err = assert_python_ok('-c', """if 1: diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 6ac584a08d1e86..0eab3f523dc5fe 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -1,5 +1,5 @@ import difflib -from test.support import findfile +from test.support import findfile, force_colorized import unittest import doctest import sys @@ -355,6 +355,22 @@ def test_range_format_context(self): self.assertEqual(fmt(3,6), '4,6') self.assertEqual(fmt(0,0), '0') + @force_colorized + def test_unified_diff_colored_output(self): + args = [['one', 'three'], ['two', 'three'], 'Original', 'Current', + '2005-01-26 23:30:50', '2010-04-02 10:20:52'] + actual = list(difflib.unified_diff(*args, lineterm='', color=True)) + + expect = [ + "\033[1m--- Original\t2005-01-26 23:30:50\033[0m", + "\033[1m+++ Current\t2010-04-02 10:20:52\033[0m", + "\033[36m@@ -1,2 +1,2 @@\033[0m", + "\033[31m-one\033[0m", + "\033[32m+two\033[0m", + "\033[0m three\033[0m", + ] + self.assertEqual(expect, actual) + class TestBytes(unittest.TestCase): # don't really care about the content of the output, just the fact diff --git a/Lib/test/test_finalization.py b/Lib/test/test_finalization.py index 42871f8a09b16b..9dd68cf8d57413 100644 --- a/Lib/test/test_finalization.py +++ b/Lib/test/test_finalization.py @@ -174,7 +174,7 @@ def test_simple(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) @@ -188,12 +188,12 @@ def test_simple_resurrect(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors(ids) - self.assertIsNot(wr(), None) + self.assertIsNotNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) @support.cpython_only def test_non_gc(self): @@ -265,7 +265,7 @@ def test_simple(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) @@ -276,19 +276,24 @@ def test_simple_resurrect(self): s = SelfCycleResurrector() ids = [id(s)] wr = weakref.ref(s) + wrc = weakref.ref(s, lambda x: None) del s gc.collect() self.assert_del_calls(ids) self.assert_survivors(ids) - # XXX is this desirable? - self.assertIs(wr(), None) + # This used to be None because weakrefs were cleared before + # calling finalizers. Now they are cleared after. + self.assertIsNotNone(wr()) + # A weakref with a callback is still cleared before calling + # finalizers. + self.assertIsNone(wrc()) # When trying to destroy the object a second time, __del__ # isn't called anymore (and the object isn't resurrected). self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) def test_simple_suicide(self): # Test the GC is able to deal with an object that kills its last @@ -301,11 +306,11 @@ def test_simple_suicide(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) class ChainedBase: @@ -378,18 +383,27 @@ def check_non_resurrecting_chain(self, classes): def check_resurrecting_chain(self, classes): N = len(classes) + def dummy_callback(ref): + pass with SimpleBase.test(): nodes = self.build_chain(classes) N = len(nodes) ids = [id(s) for s in nodes] survivor_ids = [id(s) for s in nodes if isinstance(s, SimpleResurrector)] wrs = [weakref.ref(s) for s in nodes] + wrcs = [weakref.ref(s, dummy_callback) for s in nodes] del nodes gc.collect() self.assert_del_calls(ids) self.assert_survivors(survivor_ids) - # XXX desirable? - self.assertEqual([wr() for wr in wrs], [None] * N) + for wr in wrs: + # These values used to be None because weakrefs were cleared + # before calling finalizers. Now they are cleared after. + self.assertIsNotNone(wr()) + for wr in wrcs: + # Weakrefs with callbacks are still cleared before calling + # finalizers. + self.assertIsNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) @@ -491,7 +505,7 @@ def test_legacy(self): self.assert_del_calls(ids) self.assert_tp_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_tp_del_calls(ids) @@ -507,13 +521,13 @@ def test_legacy_resurrect(self): self.assert_tp_del_calls(ids) self.assert_survivors(ids) # weakrefs are cleared before tp_del is called. - self.assertIs(wr(), None) + self.assertIsNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_tp_del_calls(ids * 2) self.assert_survivors(ids) - self.assertIs(wr(), None) + self.assertIsNone(wr()) def test_legacy_self_cycle(self): # Self-cycles with legacy finalizers end up in gc.garbage. @@ -527,11 +541,11 @@ def test_legacy_self_cycle(self): self.assert_tp_del_calls([]) self.assert_survivors([]) self.assert_garbage(ids) - self.assertIsNot(wr(), None) + self.assertIsNotNone(wr()) # Break the cycle to allow collection gc.garbage[0].ref = None self.assert_garbage([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) if __name__ == "__main__": diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index ae996e7db3c7fd..2d995751005d71 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -127,6 +127,20 @@ class ClassB(Base): obj.__class__ = ClassB + def test_name_change(self): + class Foo: + pass + + def writer(): + for _ in range(1000): + Foo.__name__ = 'Bar' + + def reader(): + for _ in range(1000): + Foo.__name__ + + self.run_one(writer, reader) + def run_one(self, writer_func, reader_func): barrier = threading.Barrier(NTHREADS) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index b41e02c3a16379..41cefe0e286d50 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1831,6 +1831,34 @@ def test_newlines_in_format_specifiers(self): for case in valid_cases: compile(case, "", "exec") + def test_raw_fstring_format_spec(self): + # Test raw f-string format spec behavior (Issue #137314). + # + # Raw f-strings should preserve literal backslashes in format specifications, + # not interpret them as escape sequences. + class UnchangedFormat: + """Test helper that returns the format spec unchanged.""" + def __format__(self, format): + return format + + # Test basic escape sequences + self.assertEqual(f"{UnchangedFormat():\xFF}", 'ÿ') + self.assertEqual(rf"{UnchangedFormat():\xFF}", '\\xFF') + + # Test nested expressions with raw/non-raw combinations + self.assertEqual(rf"{UnchangedFormat():{'\xFF'}}", 'ÿ') + self.assertEqual(f"{UnchangedFormat():{r'\xFF'}}", '\\xFF') + self.assertEqual(rf"{UnchangedFormat():{r'\xFF'}}", '\\xFF') + + # Test continuation character in format specs + self.assertEqual(f"""{UnchangedFormat():{'a'\ + 'b'}}""", 'ab') + self.assertEqual(rf"""{UnchangedFormat():{'a'\ + 'b'}}""", 'ab') + + # Test multiple format specs in same raw f-string + self.assertEqual(rf"{UnchangedFormat():\xFF} {UnchangedFormat():\n}", '\\xFF \\n') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 96ebb23f73de9d..7c9adf3049a131 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -309,6 +309,26 @@ def func(): self.assertRegex(stdout, rb"""\A\s*func=None""") self.assertFalse(stderr) + def test_datetime_weakref_cycle(self): + # https://github.com/python/cpython/issues/132413 + # If the weakref used by the datetime extension gets cleared by the GC (due to being + # in an unreachable cycle) then datetime functions would crash (get_module_state() + # was returning a NULL pointer). This bug is fixed by clearing weakrefs without + # callbacks *after* running finalizers. + code = """if 1: + import _datetime + class C: + def __del__(self): + print('__del__ called') + _datetime.timedelta(days=1) # crash? + + l = [C()] + l.append(l) + """ + rc, stdout, stderr = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(stdout.strip(), b'__del__ called') + @refcount_test def test_frame(self): def f(): @@ -658,9 +678,8 @@ def callback(ignored): gc.collect() self.assertEqual(len(ouch), 2) # else the callbacks didn't run for x in ouch: - # If the callback resurrected one of these guys, the instance - # would be damaged, with an empty __dict__. - self.assertEqual(x, None) + # The weakref should be cleared before executing the callback. + self.assertIsNone(x) def test_bug21435(self): # This is a poor test - its only virtue is that it happened to @@ -1136,6 +1155,37 @@ def test_something(self): """) assert_python_ok("-c", source) + def test_do_not_cleanup_type_subclasses_before_finalization(self): + # See https://github.com/python/cpython/issues/135552 + # If we cleanup weakrefs for tp_subclasses before calling + # the finalizer (__del__) then the line `fail = BaseNode.next.next` + # should fail because we are trying to access a subclass + # attribute. But subclass type cache was not properly invalidated. + code = """ + class BaseNode: + def __del__(self): + BaseNode.next = BaseNode.next.next + fail = BaseNode.next.next + + class Node(BaseNode): + pass + + BaseNode.next = Node() + BaseNode.next.next = Node() + """ + # this test checks garbage collection while interp + # finalization + assert_python_ok("-c", textwrap.dedent(code)) + + code_inside_function = textwrap.dedent(F""" + def test(): + {textwrap.indent(code, ' ')} + + test() + """) + # this test checks regular garbage collection + assert_python_ok("-c", code_inside_function) + class IncrementalGCTests(unittest.TestCase): @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") @@ -1335,6 +1385,7 @@ def setUp(self): def tearDown(self): gc.disable() + @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") def test_bug1055820c(self): # Corresponds to temp2c.py in the bug report. This is pretty # elaborate. @@ -1410,6 +1461,7 @@ def callback(ignored): self.assertEqual(x, None) @gc_threshold(1000, 0, 0) + @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") def test_bug1055820d(self): # Corresponds to temp2d.py in the bug report. This is very much like # test_bug1055820c, but uses a __del__ method instead of a weakref diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index d0ed5129253cdf..9e4e82b2542c15 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -4,7 +4,6 @@ import shutil import sys import unittest -import warnings from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, @@ -393,36 +392,6 @@ def test_glob_many_open_files(self): for it in iters: self.assertEqual(next(it), p) - def test_glob0(self): - with self.assertWarns(DeprecationWarning): - glob.glob0(self.tempdir, 'a') - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - eq = self.assertSequencesEqual_noorder - eq(glob.glob0(self.tempdir, 'a'), ['a']) - eq(glob.glob0(self.tempdir, '.bb'), ['.bb']) - eq(glob.glob0(self.tempdir, '.b*'), []) - eq(glob.glob0(self.tempdir, 'b'), []) - eq(glob.glob0(self.tempdir, '?'), []) - eq(glob.glob0(self.tempdir, '*a'), []) - eq(glob.glob0(self.tempdir, 'a*'), []) - - def test_glob1(self): - with self.assertWarns(DeprecationWarning): - glob.glob1(self.tempdir, 'a') - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - eq = self.assertSequencesEqual_noorder - eq(glob.glob1(self.tempdir, 'a'), ['a']) - eq(glob.glob1(self.tempdir, '.bb'), ['.bb']) - eq(glob.glob1(self.tempdir, '.b*'), ['.bb']) - eq(glob.glob1(self.tempdir, 'b'), []) - eq(glob.glob1(self.tempdir, '?'), ['a']) - eq(glob.glob1(self.tempdir, '*a'), ['a', 'aaa']) - eq(glob.glob1(self.tempdir, 'a*'), ['a', 'aaa', 'aab']) - def test_translate_matching(self): match = re.compile(glob.translate('*')).match self.assertIsNotNone(match('foo')) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index b2b64a76a9f0f6..33845d8a9e2651 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -27,24 +27,17 @@ from http.client import HTTPException -default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'} +default_builtin_hashes = {'md5', 'sha1', 'sha2', 'sha3', 'blake2'} # --with-builtin-hashlib-hashes override builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES") if builtin_hashes is None: builtin_hashes = default_builtin_hashes else: - builtin_hashes = { - m.strip() for m in builtin_hashes.strip('"').lower().split(",") - } + builtin_hash_names = builtin_hashes.strip('"').lower().split(",") + builtin_hashes = set(map(str.strip, builtin_hash_names)) -# hashlib with and without OpenSSL backend for PBKDF2 -# only import builtin_hashlib when all builtin hashes are available. -# Otherwise import prints noise on stderr +# Public 'hashlib' module with OpenSSL backend for PBKDF2. openssl_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) -if builtin_hashes == default_builtin_hashes: - builtin_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) -else: - builtin_hashlib = None try: from _hashlib import HASH, HASHXOF, openssl_md_meth_names, get_fips_mode diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 2fbc142de2fd34..c2ed30831b2e0e 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -48,6 +48,29 @@ def test_basic(self): 'Set-Cookie: d=r', 'Set-Cookie: f=h' )) + }, + + # gh-92936: allow double quote in cookie values + { + 'data': 'cookie="{"key": "value"}"', + 'dict': {'cookie': '{"key": "value"}'}, + 'repr': "", + 'output': 'Set-Cookie: cookie="{"key": "value"}"', + }, + { + 'data': 'key="some value; surrounded by quotes"', + 'dict': {'key': 'some value; surrounded by quotes'}, + 'repr': "", + 'output': 'Set-Cookie: key="some value; surrounded by quotes"', + }, + { + 'data': 'session="user123"; preferences="{"theme": "dark"}"', + 'dict': {'session': 'user123', 'preferences': '{"theme": "dark"}'}, + 'repr': "", + 'output': '\n'.join(( + 'Set-Cookie: preferences="{"theme": "dark"}"', + 'Set-Cookie: session="user123"', + )) } ] diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 30e01b8cd87a75..c8c1a5226114b9 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -5831,6 +5831,21 @@ def test_collections_abc_module_has_signatures(self): import collections.abc self._test_module_has_signatures(collections.abc) + def test_datetime_module_has_signatures(self): + # Only test if the C implementation is available. + import_helper.import_module('_datetime') + import datetime + no_signature = {'tzinfo'} + unsupported_signature = {'timezone'} + methods_unsupported_signature = { + 'date': {'replace'}, + 'time': {'replace'}, + 'datetime': {'replace', 'combine'}, + } + self._test_module_has_signatures(datetime, + no_signature, unsupported_signature, + methods_unsupported_signature=methods_unsupported_signature) + def test_errno_module_has_signatures(self): import errno self._test_module_has_signatures(errno) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index b487bcabf01ca4..92be2763e5ed1e 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -808,7 +808,12 @@ def test_closefd_attr(self): def test_garbage_collection(self): # FileIO objects are collected, and collecting them flushes # all data to disk. - with warnings_helper.check_warnings(('', ResourceWarning)): + # + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) f = self.FileIO(os_helper.TESTFN, "wb") f.write(b"abcxxx") f.f = f @@ -1809,7 +1814,11 @@ def test_garbage_collection(self): # C BufferedReader objects are collected. # The Python version has __del__, so it ends into gc.garbage instead self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with warnings_helper.check_warnings(('', ResourceWarning)): + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) rawio = self.FileIO(os_helper.TESTFN, "w+b") f = self.tp(rawio) f.f = f @@ -2158,7 +2167,11 @@ def test_garbage_collection(self): # all data to disk. # The Python version has __del__, so it ends into gc.garbage instead self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with warnings_helper.check_warnings(('', ResourceWarning)): + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) rawio = self.FileIO(os_helper.TESTFN, "w+b") f = self.tp(rawio) f.write(b"123xxx") @@ -4080,7 +4093,8 @@ def test_garbage_collection(self): # C TextIOWrapper objects are collected, and collecting them flushes # all data to disk. # The Python version has __del__, so it ends in gc.garbage instead. - with warnings_helper.check_warnings(('', ResourceWarning)): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) rawio = self.FileIO(os_helper.TESTFN, "wb") b = self.BufferedWriter(rawio) t = self.TextIOWrapper(b, encoding="ascii") diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 55b502e52ca454..698e137e3e8abd 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -5,6 +5,7 @@ from unittest import mock import unittest import locale +import os import sys import codecs @@ -486,6 +487,54 @@ def test_japanese(self): self.check('jp_jp', 'ja_JP.eucJP') +class TestRealLocales(unittest.TestCase): + def setUp(self): + oldlocale = locale.setlocale(locale.LC_CTYPE) + self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) + + def test_getsetlocale_issue1813(self): + # Issue #1813: setting and getting the locale under a Turkish locale + try: + locale.setlocale(locale.LC_CTYPE, 'tr_TR') + except locale.Error: + # Unsupported locale on this system + self.skipTest('test needs Turkish locale') + loc = locale.getlocale(locale.LC_CTYPE) + if verbose: + print('testing with %a' % (loc,), end=' ', flush=True) + try: + locale.setlocale(locale.LC_CTYPE, loc) + except locale.Error as exc: + # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) + # and the tr_TR locale on Windows. getlocale() builds a locale + # which is not recognize by setlocale(). + self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") + self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) + + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + def test_setlocale_long_encoding(self): + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'English.%016d' % 1252) + locale.setlocale(locale.LC_CTYPE, 'English.%015d' % 1252) + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.1252', loc) + loc2 = loc.replace('.1252', '.%016d' % 1252, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + loc2 = loc.replace('.1252', '.%015d' % 1252, 1) + locale.setlocale(locale.LC_ALL, loc2) + + # gh-137273: Debug assertion failure on Windows for long encoding. + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'en_US.' + 'x'*16) + locale.setlocale(locale.LC_CTYPE, 'en_US.UTF-8') + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.UTF-8', loc) + loc2 = loc.replace('.UTF-8', '.' + 'x'*16, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + + class TestMiscellaneous(unittest.TestCase): def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is @@ -552,27 +601,6 @@ def test_setlocale_category(self): # crasher from bug #7419 self.assertRaises(locale.Error, locale.setlocale, 12345) - def test_getsetlocale_issue1813(self): - # Issue #1813: setting and getting the locale under a Turkish locale - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') - loc = locale.getlocale(locale.LC_CTYPE) - if verbose: - print('testing with %a' % (loc,), end=' ', flush=True) - try: - locale.setlocale(locale.LC_CTYPE, loc) - except locale.Error as exc: - # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) - # and the tr_TR locale on Windows. getlocale() builds a locale - # which is not recognize by setlocale(). - self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") - self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) - def test_invalid_locale_format_in_localetuple(self): with self.assertRaises(TypeError): locale.setlocale(locale.LC_ALL, b'fi_FI') diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index a932ac80117d27..4122f786a9afb0 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -3,6 +3,7 @@ import collections import dis import functools +import inspect import math import operator import sys @@ -1709,6 +1710,27 @@ def func(v=1): ('branch right', 'func', 6, 8), ('branch right', 'func', 2, 10)]) + def test_callback_set_frame_lineno(self): + def func(s: str) -> int: + if s.startswith("t"): + return 1 + else: + return 0 + + def callback(code, from_, to): + # try set frame.f_lineno + frame = inspect.currentframe() + while frame and frame.f_code is not code: + frame = frame.f_back + + self.assertIsNotNone(frame) + frame.f_lineno = frame.f_lineno + 1 # run next instruction + + sys.monitoring.set_local_events(TEST_TOOL, func.__code__, E.BRANCH_LEFT) + sys.monitoring.register_callback(TEST_TOOL, E.BRANCH_LEFT, callback) + + self.assertEqual(func("true"), 1) + class TestBranchConsistency(MonitoringTestBase, unittest.TestCase): diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 0207843cc0e8f7..13424991639215 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -162,48 +162,55 @@ def baz(): @unittest.skipIf(support.check_bolt_optimized(), "fails on BOLT instrumented binaries") def test_sys_api(self): - code = """if 1: - import sys - def foo(): - pass - - def spam(): - pass + for define_eval_hook in (False, True): + code = """if 1: + import sys + def foo(): + pass - def bar(): - sys.deactivate_stack_trampoline() - foo() - sys.activate_stack_trampoline("perf") - spam() + def spam(): + pass - def baz(): - bar() + def bar(): + sys.deactivate_stack_trampoline() + foo() + sys.activate_stack_trampoline("perf") + spam() - sys.activate_stack_trampoline("perf") - baz() - """ - with temp_dir() as script_dir: - script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} - with subprocess.Popen( - [sys.executable, script], - text=True, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - env=env, - ) as process: - stdout, stderr = process.communicate() + def baz(): + bar() - self.assertEqual(stderr, "") - self.assertEqual(stdout, "") + sys.activate_stack_trampoline("perf") + baz() + """ + if define_eval_hook: + set_eval_hook = """if 1: + import _testinternalcapi + _testinternalcapi.set_eval_frame_record([]) +""" + code = set_eval_hook + code + with temp_dir() as script_dir: + script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} + with subprocess.Popen( + [sys.executable, script], + text=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env=env, + ) as process: + stdout, stderr = process.communicate() - perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") - self.assertTrue(perf_file.exists()) - perf_file_contents = perf_file.read_text() - self.assertNotIn(f"py::foo:{script}", perf_file_contents) - self.assertIn(f"py::spam:{script}", perf_file_contents) - self.assertIn(f"py::bar:{script}", perf_file_contents) - self.assertIn(f"py::baz:{script}", perf_file_contents) + self.assertEqual(stderr, "") + self.assertEqual(stdout, "") + + perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") + self.assertTrue(perf_file.exists()) + perf_file_contents = perf_file.read_text() + self.assertNotIn(f"py::foo:{script}", perf_file_contents) + self.assertIn(f"py::spam:{script}", perf_file_contents) + self.assertIn(f"py::bar:{script}", perf_file_contents) + self.assertIn(f"py::baz:{script}", perf_file_contents) def test_sys_api_with_existing_trampoline(self): code = """if 1: diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index d23d3623235f38..fe05224828bd27 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -14,89 +14,154 @@ class ResourceTest(unittest.TestCase): def test_args(self): self.assertRaises(TypeError, resource.getrlimit) - self.assertRaises(TypeError, resource.getrlimit, 42, 42) + self.assertRaises(TypeError, resource.getrlimit, 0, 42) + self.assertRaises(OverflowError, resource.getrlimit, 2**1000) + self.assertRaises(OverflowError, resource.getrlimit, -2**1000) + self.assertRaises(TypeError, resource.getrlimit, '0') self.assertRaises(TypeError, resource.setrlimit) - self.assertRaises(TypeError, resource.setrlimit, 42, 42, 42) + self.assertRaises(TypeError, resource.setrlimit, 0) + self.assertRaises(TypeError, resource.setrlimit, 0, 42) + self.assertRaises(TypeError, resource.setrlimit, 0, 42, 42) + self.assertRaises(OverflowError, resource.setrlimit, 2**1000, (42, 42)) + self.assertRaises(OverflowError, resource.setrlimit, -2**1000, (42, 42)) + self.assertRaises(ValueError, resource.setrlimit, 0, (42,)) + self.assertRaises(ValueError, resource.setrlimit, 0, (42, 42, 42)) + self.assertRaises(TypeError, resource.setrlimit, '0', (42, 42)) + self.assertRaises(TypeError, resource.setrlimit, 0, ('42', 42)) + self.assertRaises(TypeError, resource.setrlimit, 0, (42, '42')) @unittest.skipIf(sys.platform == "vxworks", "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') def test_fsize_ismax(self): - try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: - pass - else: - # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really big - # number on a platform with large file support. On these platforms, - # we need to test that the get/setrlimit functions properly convert - # the number to a C long long and that the conversion doesn't raise - # an error. - self.assertEqual(resource.RLIM_INFINITY, max) - resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really big + # number on a platform with large file support. On these platforms, + # we need to test that the get/setrlimit functions properly convert + # the number to a C long long and that the conversion doesn't raise + # an error. + self.assertEqual(resource.RLIM_INFINITY, max) + resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') def test_fsize_enforced(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + # Check to see what happens when the RLIMIT_FSIZE is small. Some + # versions of Python were terminated by an uncaught SIGXFSZ, but + # pythonrun.c has been fixed to ignore that exception. If so, the + # write() should return EFBIG when the limit is exceeded. + + # At least one platform has an unlimited RLIMIT_FSIZE and attempts + # to change it raise ValueError instead. try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: - pass - else: - # Check to see what happens when the RLIMIT_FSIZE is small. Some - # versions of Python were terminated by an uncaught SIGXFSZ, but - # pythonrun.c has been fixed to ignore that exception. If so, the - # write() should return EFBIG when the limit is exceeded. - - # At least one platform has an unlimited RLIMIT_FSIZE and attempts - # to change it raise ValueError instead. try: + resource.setrlimit(resource.RLIMIT_FSIZE, (1024, max)) + limit_set = True + except ValueError: + limit_set = False + f = open(os_helper.TESTFN, "wb") + try: + f.write(b"X" * 1024) try: - resource.setrlimit(resource.RLIMIT_FSIZE, (1024, max)) - limit_set = True - except ValueError: - limit_set = False - f = open(os_helper.TESTFN, "wb") - try: - f.write(b"X" * 1024) - try: - f.write(b"Y") + f.write(b"Y") + f.flush() + # On some systems (e.g., Ubuntu on hppa) the flush() + # doesn't always cause the exception, but the close() + # does eventually. Try flushing several times in + # an attempt to ensure the file is really synced and + # the exception raised. + for i in range(5): + time.sleep(.1) f.flush() - # On some systems (e.g., Ubuntu on hppa) the flush() - # doesn't always cause the exception, but the close() - # does eventually. Try flushing several times in - # an attempt to ensure the file is really synced and - # the exception raised. - for i in range(5): - time.sleep(.1) - f.flush() - except OSError: - if not limit_set: - raise - if limit_set: - # Close will attempt to flush the byte we wrote - # Restore limit first to avoid getting a spurious error - resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) - finally: - f.close() - finally: + except OSError: + if not limit_set: + raise if limit_set: + # Close will attempt to flush the byte we wrote + # Restore limit first to avoid getting a spurious error resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) - os_helper.unlink(os_helper.TESTFN) + finally: + f.close() + finally: + if limit_set: + resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + os_helper.unlink(os_helper.TESTFN) - def test_fsize_toobig(self): + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_too_big(self): # Be sure that setrlimit is checking for really large values too_big = 10**50 + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + try: + resource.setrlimit(resource.RLIMIT_FSIZE, (too_big, max)) + except (OverflowError, ValueError): + pass try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: + resource.setrlimit(resource.RLIMIT_FSIZE, (max, too_big)) + except (OverflowError, ValueError): pass + + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_not_too_big(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + self.addCleanup(resource.setrlimit, resource.RLIMIT_FSIZE, (cur, max)) + + def expected(cur): + if resource.RLIM_INFINITY < 0: + return [(cur, max), (resource.RLIM_INFINITY, max)] + elif resource.RLIM_INFINITY < cur: + return [(resource.RLIM_INFINITY, max)] + else: + return [(cur, max)] + + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31-5, max)) + self.assertEqual(resource.getrlimit(resource.RLIMIT_FSIZE), (2**31-5, max)) + + try: + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32, max)) + except OverflowError: + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**31)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**32-5)) else: + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**32)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31, max)) + self.assertEqual(resource.getrlimit(resource.RLIMIT_FSIZE), (2**31, max)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32-5, max)) + self.assertEqual(resource.getrlimit(resource.RLIMIT_FSIZE), (2**32-5, max)) + + resource.setrlimit(resource.RLIMIT_FSIZE, (2**63-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**63-5)) try: - resource.setrlimit(resource.RLIMIT_FSIZE, (too_big, max)) - except (OverflowError, ValueError): - pass - try: - resource.setrlimit(resource.RLIMIT_FSIZE, (max, too_big)) - except (OverflowError, ValueError): + resource.setrlimit(resource.RLIMIT_FSIZE, (2**63, max)) + except ValueError: + # There is a hard limit on macOS. pass + else: + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**63)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**64-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**64-5)) + + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_negative(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + for value in -5, -2**31, -2**32-5, -2**63, -2**64-5, -2**1000: + with self.subTest(value=value): + # This test assumes that the values don't map to RLIM_INFINITY, + # though Posix doesn't guarantee it. + self.assertNotEqual(value, resource.RLIM_INFINITY) + + self.assertRaises(ValueError, resource.setrlimit, resource.RLIMIT_FSIZE, (value, max)) + self.assertRaises(ValueError, resource.setrlimit, resource.RLIMIT_FSIZE, (cur, value)) @unittest.skipUnless(hasattr(resource, "getrusage"), "needs getrusage") def test_getrusage(self): @@ -117,21 +182,18 @@ def test_getrusage(self): # Issue 6083: Reference counting bug @unittest.skipIf(sys.platform == "vxworks", "setting RLIMIT_CPU is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_CPU'), 'requires resource.RLIMIT_CPU') def test_setrusage_refcount(self): - try: - limits = resource.getrlimit(resource.RLIMIT_CPU) - except AttributeError: - pass - else: - class BadSequence: - def __len__(self): - return 2 - def __getitem__(self, key): - if key in (0, 1): - return len(tuple(range(1000000))) - raise IndexError + limits = resource.getrlimit(resource.RLIMIT_CPU) + class BadSequence: + def __len__(self): + return 2 + def __getitem__(self, key): + if key in (0, 1): + return len(tuple(range(1000000))) + raise IndexError - resource.setrlimit(resource.RLIMIT_CPU, BadSequence()) + resource.setrlimit(resource.RLIMIT_CPU, BadSequence()) def test_pagesize(self): pagesize = resource.getpagesize() @@ -168,7 +230,8 @@ class BadSeq: def __len__(self): return 2 def __getitem__(self, key): - return limits[key] - 1 # new reference + lim = limits[key] + return lim - 1 if lim > 0 else lim + sys.maxsize*2 # new reference limits = resource.getrlimit(resource.RLIMIT_AS) self.assertEqual(resource.prlimit(0, resource.RLIMIT_AS, BadSeq()), diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 6d62d6119255a8..d6cc22558ec4fa 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -14,7 +14,7 @@ import unittest from test import support from test.support import ( - is_apple, is_apple_mobile, os_helper, threading_helper + force_not_colorized, is_apple, is_apple_mobile, os_helper, threading_helper ) from test.support.script_helper import assert_python_ok, spawn_python try: @@ -353,6 +353,7 @@ def check_signum(signals): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + @force_not_colorized def test_wakeup_write_error(self): # Issue #16105: write() errors in the C signal handler should not # pass silently. diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 3dd67b2a2aba97..76fd33c7dc8767 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1538,6 +1538,40 @@ def testSetSockOpt(self): reuse = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) self.assertFalse(reuse == 0, "failed to set reuse mode") + def test_setsockopt_errors(self): + # See issue #107546. + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # No error expected. + + with self.assertRaises(OverflowError): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 2 ** 100) + + with self.assertRaises(OverflowError): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, - 2 ** 100) + + with self.assertRaises(OverflowError): + sock.setsockopt(socket.SOL_SOCKET, 2 ** 100, 1) + + with self.assertRaises(OverflowError): + sock.setsockopt(2 ** 100, socket.SO_REUSEADDR, 1) + + with self.assertRaisesRegex(TypeError, "socket option should be int, bytes-like object or None"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, dict()) + + with self.assertRaisesRegex(TypeError, "requires 4 arguments when the third argument is None"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, None) + + with self.assertRaisesRegex(TypeError, "only takes 4 arguments when the third argument is None"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1, 2) + + with self.assertRaisesRegex(TypeError, "takes at least 3 arguments"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) + + with self.assertRaisesRegex(TypeError, "takes at most 4 arguments"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1, 2, 3) + def testSendAfterClose(self): # testing send() after close() with timeout with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index c8939383c75d6d..b5263129baed3f 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -49,6 +49,7 @@ HOST = socket_helper.HOST IS_OPENSSL_3_0_0 = ssl.OPENSSL_VERSION_INFO >= (3, 0, 0) CAN_GET_SELECTED_OPENSSL_GROUP = ssl.OPENSSL_VERSION_INFO >= (3, 2) +CAN_IGNORE_UNKNOWN_OPENSSL_GROUPS = ssl.OPENSSL_VERSION_INFO >= (3, 3) CAN_GET_AVAILABLE_OPENSSL_GROUPS = ssl.OPENSSL_VERSION_INFO >= (3, 5) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') @@ -964,8 +965,14 @@ def test_get_ciphers(self): def test_set_groups(self): ctx = ssl.create_default_context() - self.assertIsNone(ctx.set_groups('P-256:X25519')) - self.assertRaises(ssl.SSLError, ctx.set_groups, 'P-256:xxx') + # We use P-256 and P-384 (FIPS 186-4) that are alloed by OpenSSL + # even if FIPS module is enabled. Ignoring unknown groups is only + # supported since OpenSSL 3.3. + self.assertIsNone(ctx.set_groups('P-256:P-384')) + + self.assertRaises(ssl.SSLError, ctx.set_groups, 'P-256:foo') + if CAN_IGNORE_UNKNOWN_OPENSSL_GROUPS: + self.assertIsNone(ctx.set_groups('P-256:?foo')) @unittest.skipUnless(CAN_GET_AVAILABLE_OPENSSL_GROUPS, "OpenSSL version doesn't support getting groups") diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 486bf10a0b5647..f89237931b7185 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1340,6 +1340,7 @@ def test_disable_gil_abi(self): @test.support.cpython_only +@force_not_colorized class UnraisableHookTest(unittest.TestCase): def test_original_unraisablehook(self): _testcapi = import_helper.import_module('_testcapi') diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 28914df6b010d0..860413b88eb6b5 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -841,6 +841,57 @@ def test_next_on_empty_tarfile(self): with tarfile.open(fileobj=fd, mode="r") as tf: self.assertEqual(tf.next(), None) + def _setup_symlink_to_target(self, temp_dirpath): + target_filepath = os.path.join(temp_dirpath, "target") + ustar_dirpath = os.path.join(temp_dirpath, "ustar") + hardlink_filepath = os.path.join(ustar_dirpath, "lnktype") + with open(target_filepath, "wb") as f: + f.write(b"target") + os.makedirs(ustar_dirpath) + os.symlink(target_filepath, hardlink_filepath) + return target_filepath, hardlink_filepath + + def _assert_on_file_content(self, filepath, digest): + with open(filepath, "rb") as f: + data = f.read() + self.assertEqual(sha256sum(data), digest) + + @unittest.skipUnless( + hasattr(os, "link"), "Missing hardlink implementation" + ) + @os_helper.skip_unless_symlink + def test_extract_hardlink_on_symlink(self): + """ + This test verifies that extracting a hardlink will not follow an + existing symlink after a FileExistsError on os.link. + """ + with os_helper.temp_dir() as DIR: + target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR) + with tarfile.open(tarname, encoding="iso8859-1") as tar: + tar.extract("ustar/regtype", DIR, filter="data") + tar.extract("ustar/lnktype", DIR, filter="data") + self._assert_on_file_content(target_filepath, sha256sum(b"target")) + self._assert_on_file_content(hardlink_filepath, sha256_regtype) + + @unittest.skipUnless( + hasattr(os, "link"), "Missing hardlink implementation" + ) + @os_helper.skip_unless_symlink + def test_extractall_hardlink_on_symlink(self): + """ + This test verifies that extracting a hardlink will not follow an + existing symlink after a FileExistsError on os.link. + """ + with os_helper.temp_dir() as DIR: + target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR) + with tarfile.open(tarname, encoding="iso8859-1") as tar: + tar.extractall( + DIR, members=["ustar/regtype", "ustar/lnktype"], filter="data", + ) + self._assert_on_file_content(target_filepath, sha256sum(b"target")) + self._assert_on_file_content(hardlink_filepath, sha256_regtype) + + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None @@ -1737,6 +1788,16 @@ def test_file_mode(self): finally: os.umask(original_umask) + def test_pathlike_name(self): + expected_name = os.path.abspath(tmpname) + tarpath = os_helper.FakePath(tmpname) + + for func in (tarfile.open, tarfile.TarFile.open): + with self.subTest(): + with func(tarpath, self.mode) as tar: + self.assertEqual(tar.name, expected_name) + os_helper.unlink(tmpname) + class GzipStreamWriteTest(GzipTest, StreamWriteTest): def test_source_directory_not_leaked(self): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 002a1feeb85c94..0ba78b9a1807d2 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -2494,6 +2494,7 @@ def test_atexit_called_once(self): self.assertFalse(err) + @force_not_colorized def test_atexit_after_shutdown(self): # The only way to do this is by registering an atexit within # an atexit, which is intended to raise an exception. diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b1615bbff383c2..6317d4657619f0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3958,6 +3958,7 @@ class C: pass def test_defining_generic_protocols(self): T = TypeVar('T') + T2 = TypeVar('T2') S = TypeVar('S') @runtime_checkable @@ -3967,17 +3968,26 @@ def meth(self): pass class P(PR[int, T], Protocol[T]): y = 1 + self.assertEqual(P.__parameters__, (T,)) + with self.assertRaises(TypeError): PR[int] with self.assertRaises(TypeError): P[int, str] + with self.assertRaisesRegex( + TypeError, + re.escape('Some type variables (~S) are not listed in Protocol[~T, ~T2]'), + ): + class ExtraTypeVars(P[S], Protocol[T, T2]): ... class C(PR[int, T]): pass + self.assertEqual(C.__parameters__, (T,)) self.assertIsInstance(C[str](), C) def test_defining_generic_protocols_old_style(self): T = TypeVar('T') + T2 = TypeVar('T2') S = TypeVar('S') @runtime_checkable @@ -3996,9 +4006,19 @@ class P(PR[int, str], Protocol): class P1(Protocol, Generic[T]): def bar(self, x: T) -> str: ... + self.assertEqual(P1.__parameters__, (T,)) + class P2(Generic[T], Protocol): def bar(self, x: T) -> str: ... + self.assertEqual(P2.__parameters__, (T,)) + + msg = re.escape('Some type variables (~S) are not listed in Protocol[~T, ~T2]') + with self.assertRaisesRegex(TypeError, msg): + class ExtraTypeVars(P1[S], Protocol[T, T2]): ... + with self.assertRaisesRegex(TypeError, msg): + class ExtraTypeVars(P2[S], Protocol[T, T2]): ... + @runtime_checkable class PSub(P1[str], Protocol): x = 1 @@ -4011,6 +4031,28 @@ def bar(self, x: str) -> str: self.assertIsInstance(Test(), PSub) + def test_protocol_parameter_order(self): + # https://github.com/python/cpython/issues/137191 + T1 = TypeVar("T1") + T2 = TypeVar("T2", default=object) + + class A(Protocol[T1]): ... + + class B0(A[T2], Generic[T1, T2]): ... + self.assertEqual(B0.__parameters__, (T1, T2)) + + class B1(A[T2], Protocol, Generic[T1, T2]): ... + self.assertEqual(B1.__parameters__, (T1, T2)) + + class B2(A[T2], Protocol[T1, T2]): ... + self.assertEqual(B2.__parameters__, (T1, T2)) + + class B3[T1, T2](A[T2], Protocol): + @staticmethod + def get_typeparams(): + return (T1, T2) + self.assertEqual(B3.__parameters__, B3.get_typeparams()) + def test_pep695_generic_protocol_callable_members(self): @runtime_checkable class Foo[T](Protocol): diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 4c7c900eb56ae1..47f6b46061ac30 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1044,6 +1044,32 @@ def callback(obj): stderr = res.err.decode("ascii", "backslashreplace") self.assertNotRegex(stderr, "_Py_Dealloc: Deallocator of type 'TestObj'") + def test_clearing_weakrefs_in_gc(self): + # This test checks that when finalizers are called: + # 1. weakrefs with callbacks have been cleared + # 2. weakrefs without callbacks have not been cleared + errors = [] + def test(): + class Class: + def __init__(self): + self._self = self + self.wr1 = weakref.ref(Class, lambda x: None) + self.wr2 = weakref.ref(Class) + + def __del__(self): + # we can't use assert* here, because gc will swallow + # exceptions + if self.wr1() is not None: + errors.append("weakref with callback as cleared") + if self.wr2() is not Class: + errors.append("weakref without callback was cleared") + + Class() + + test() + gc.collect() + self.assertEqual(errors, []) + class SubclassableWeakrefTestCase(TestBase): diff --git a/Lib/traceback.py b/Lib/traceback.py index f0dbb6352f7760..318ec13cf91121 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -137,8 +137,9 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ BUILTIN_EXCEPTION_LIMIT = object() -def _print_exception_bltin(exc, /): - file = sys.stderr if sys.stderr is not None else sys.__stderr__ +def _print_exception_bltin(exc, file=None, /): + if file is None: + file = sys.stderr if sys.stderr is not None else sys.__stderr__ colorize = _colorize.can_colorize(file=file) return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) diff --git a/Lib/typing.py b/Lib/typing.py index f1455c273d31ca..036636f7e0e6a8 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -256,16 +256,27 @@ def _type_repr(obj): return _lazy_annotationlib.type_repr(obj) -def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): +def _collect_type_parameters( + args, + *, + enforce_default_ordering: bool = True, + validate_all: bool = False, +): """Collect all type parameters in args in order of first appearance (lexicographic order). + Having an explicit `Generic` or `Protocol` base class determines + the exact parameter order. + For example:: >>> P = ParamSpec('P') >>> T = TypeVar('T') >>> _collect_type_parameters((T, Callable[P, T])) (~T, ~P) + >>> _collect_type_parameters((list[T], Generic[P, T])) + (~P, ~T) + """ # required type parameter cannot appear after parameter with default default_encountered = False @@ -297,6 +308,17 @@ def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): ' follows type parameter with a default') parameters.append(t) + elif ( + not validate_all + and isinstance(t, _GenericAlias) + and t.__origin__ in (Generic, Protocol) + ): + # If we see explicit `Generic[...]` or `Protocol[...]` base classes, + # we need to just copy them as-is. + # Unless `validate_all` is passed, in this case it means that + # we are doing a validation of `Generic` subclasses, + # then we collect all unique parameters to be able to inspect them. + parameters = t.__parameters__ else: if _is_unpacked_typevartuple(t): type_var_tuple_encountered = True @@ -1156,20 +1178,22 @@ def _generic_init_subclass(cls, *args, **kwargs): if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _collect_type_parameters(cls.__orig_bases__) + tvars = _collect_type_parameters(cls.__orig_bases__, validate_all=True) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. # Also check for and reject plain Generic, # and reject multiple Generic[...]. gvars = None + basename = None for base in cls.__orig_bases__: if (isinstance(base, _GenericAlias) and - base.__origin__ is Generic): + base.__origin__ in (Generic, Protocol)): if gvars is not None: raise TypeError( "Cannot inherit from Generic[...] multiple times.") gvars = base.__parameters__ + basename = base.__origin__.__name__ if gvars is not None: tvarset = set(tvars) gvarset = set(gvars) @@ -1177,7 +1201,7 @@ def _generic_init_subclass(cls, *args, **kwargs): s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) s_args = ', '.join(str(g) for g in gvars) raise TypeError(f"Some type variables ({s_vars}) are" - f" not listed in Generic[{s_args}]") + f" not listed in {basename}[{s_args}]") tvars = gvars cls.__parameters__ = tuple(tvars) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index b31cb766a468f4..4da6d924848889 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.49.1", - url="https://sqlite.org/2025/sqlite-autoconf-3490100.tar.gz", - checksum="106642d8ccb36c5f7323b64e4152e9b719f7c0215acf5bfeac3d5e7f97b59254", + name="SQLite 3.50.4", + url="https://www.sqlite.org/2025/sqlite-autoconf-3500400.tar.gz", + checksum="a3db587a1b92ee5ddac2f66b3edb41b26f9c867275782d46c3a088977d6a5b18", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' @@ -1747,7 +1747,7 @@ def main(): fn = os.path.join(folder, "ReadMe.rtf") patchFile("resources/ReadMe.rtf", fn) fn = os.path.join(folder, "Update Shell Profile.command") - patchScript("scripts/postflight.patch-profile", fn) + patchScript("resources/update_shell_profile.command", fn) fn = os.path.join(folder, "Install Certificates.command") patchScript("resources/install_certificates.command", fn) os.chmod(folder, STAT_0o755) diff --git a/Mac/BuildScript/resources/update_shell_profile.command b/Mac/BuildScript/resources/update_shell_profile.command new file mode 100755 index 00000000000000..3cf4d74de9f09a --- /dev/null +++ b/Mac/BuildScript/resources/update_shell_profile.command @@ -0,0 +1,116 @@ +#!/bin/sh + +echo "This script will update your shell profile when the 'bin' directory" +echo "of python is not early enough of the PATH of your shell." +echo "These changes will be effective only in shell windows that you open" +echo "after running this script." + +PYVER=@PYVER@ +PYTHON_ROOT="/Library/Frameworks/Python.framework/Versions/@PYVER@" + +if [ `id -ur` = 0 ]; then + # Run from the installer, do some trickery to fetch the information + # we need. + theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" + +else + theShell="${SHELL}" +fi + +# Make sure the directory ${PYTHON_ROOT}/bin is on the users PATH. +BSH="`basename "${theShell}"`" +case "${BSH}" in +bash|ksh|sh|*csh|zsh|fish) + if [ `id -ur` = 0 ]; then + P=`su - ${USER} -c 'echo A-X-4-X@@$PATH@@X-4-X-A' | grep 'A-X-4-X@@.*@@X-4-X-A' | sed -e 's/^A-X-4-X@@//g' -e 's/@@X-4-X-A$//g'` + else + P="`(exec -l ${theShell} -c 'echo $PATH')`" + fi + ;; +*) + echo "Sorry, I don't know how to patch $BSH shells" + exit 0 + ;; +esac + +# Now ensure that our bin directory is on $P and before /usr/bin at that +for elem in `echo $P | tr ':' ' '` +do + if [ "${elem}" = "${PYTHON_ROOT}/bin" ]; then + echo "All right, you're a python lover already" + exit 0 + elif [ "${elem}" = "/usr/bin" ]; then + break + fi +done + +echo "${PYTHON_ROOT}/bin is not on your PATH or at least not early enough" +case "${BSH}" in +*csh) + if [ -f "${HOME}/.tcshrc" ]; then + RC="${HOME}/.tcshrc" + else + RC="${HOME}/.cshrc" + fi + # Create backup copy before patching + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "" >> "${RC}" + echo "# Setting PATH for Python ${PYVER}" >> "${RC}" + echo "# The original version is saved in .cshrc.pysave" >> "${RC}" + echo "set path=(${PYTHON_ROOT}/bin "'$path'")" >> "${RC}" + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; +bash) + if [ -e "${HOME}/.bash_profile" ]; then + PR="${HOME}/.bash_profile" + elif [ -e "${HOME}/.bash_login" ]; then + PR="${HOME}/.bash_login" + elif [ -e "${HOME}/.profile" ]; then + PR="${HOME}/.profile" + else + PR="${HOME}/.bash_profile" + fi + ;; +fish) + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" + mkdir -p "$CONFIG_DIR" + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "# Setting PATH for Python ${PYVER}" > "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; +zsh) + PR="${HOME}/.zprofile" + ;; +*sh) + PR="${HOME}/.profile" + ;; +esac + +# Create backup copy before patching +if [ -f "${PR}" ]; then + cp -fp "${PR}" "${PR}.pysave" +fi +echo "" >> "${PR}" +echo "# Setting PATH for Python ${PYVER}" >> "${PR}" +echo "# The original version is saved in `basename ${PR}`.pysave" >> "${PR}" +echo 'PATH="'"${PYTHON_ROOT}/bin"':${PATH}"' >> "${PR}" +echo 'export PATH' >> "${PR}" +if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${PR}" +fi +exit 0 diff --git a/Mac/BuildScript/scripts/postflight.patch-profile b/Mac/BuildScript/scripts/postflight.patch-profile index 9caf62211ddd16..ce8720f895d1b5 100755 --- a/Mac/BuildScript/scripts/postflight.patch-profile +++ b/Mac/BuildScript/scripts/postflight.patch-profile @@ -1,116 +1,104 @@ #!/bin/sh -echo "This script will update your shell profile when the 'bin' directory" -echo "of python is not early enough of the PATH of your shell." -echo "These changes will be effective only in shell windows that you open" -echo "after running this script." - PYVER=@PYVER@ PYTHON_ROOT="/Library/Frameworks/Python.framework/Versions/@PYVER@" -if [ `id -ur` = 0 ]; then - # Run from the installer, do some trickery to fetch the information - # we need. - theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" -else - theShell="${SHELL}" -fi +# Run from the installer, do some trickery to fetch the information +# we need. +theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" # Make sure the directory ${PYTHON_ROOT}/bin is on the users PATH. BSH="`basename "${theShell}"`" case "${BSH}" in bash|ksh|sh|*csh|zsh|fish) - if [ `id -ur` = 0 ]; then - P=`su - ${USER} -c 'echo A-X-4-X@@$PATH@@X-4-X-A' | grep 'A-X-4-X@@.*@@X-4-X-A' | sed -e 's/^A-X-4-X@@//g' -e 's/@@X-4-X-A$//g'` - else - P="`(exec -l ${theShell} -c 'echo $PATH')`" - fi - ;; + true + ;; *) - echo "Sorry, I don't know how to patch $BSH shells" - exit 0 - ;; + exit 0 + ;; esac -# Now ensure that our bin directory is on $P and before /usr/bin at that -for elem in `echo $P | tr ':' ' '` -do - if [ "${elem}" = "${PYTHON_ROOT}/bin" ]; then - echo "All right, you're a python lover already" - exit 0 - elif [ "${elem}" = "/usr/bin" ]; then - break - fi -done - -echo "${PYTHON_ROOT}/bin is not on your PATH or at least not early enough" case "${BSH}" in *csh) - if [ -f "${HOME}/.tcshrc" ]; then - RC="${HOME}/.tcshrc" - else - RC="${HOME}/.cshrc" - fi - # Create backup copy before patching - if [ -f "${RC}" ]; then - cp -fp "${RC}" "${RC}.pysave" - fi - echo "" >> "${RC}" - echo "# Setting PATH for Python ${PYVER}" >> "${RC}" - echo "# The original version is saved in .cshrc.pysave" >> "${RC}" - echo "set path=(${PYTHON_ROOT}/bin "'$path'")" >> "${RC}" - if [ `id -ur` = 0 ]; then - chown "${USER}" "${RC}" - fi - exit 0 - ;; + if [ -f "${HOME}/.tcshrc" ]; then + RC="${HOME}/.tcshrc" + else + RC="${HOME}/.cshrc" + fi + + # Drop privileges while writing files. + su -m ${USER} <> "${RC}" + echo "# Setting PATH for Python ${PYVER}" >> "${RC}" + echo "# The original version is saved in .cshrc.pysave" >> "${RC}" + echo "set path=(${PYTHON_ROOT}/bin "'\$path'")" >> "${RC}" +EOFC + + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; bash) - if [ -e "${HOME}/.bash_profile" ]; then - PR="${HOME}/.bash_profile" - elif [ -e "${HOME}/.bash_login" ]; then - PR="${HOME}/.bash_login" - elif [ -e "${HOME}/.profile" ]; then - PR="${HOME}/.profile" - else - PR="${HOME}/.bash_profile" - fi - ;; + if [ -e "${HOME}/.bash_profile" ]; then + PR="${HOME}/.bash_profile" + elif [ -e "${HOME}/.bash_login" ]; then + PR="${HOME}/.bash_login" + elif [ -e "${HOME}/.profile" ]; then + PR="${HOME}/.profile" + else + PR="${HOME}/.bash_profile" + fi + ;; fish) - CONFIG_DIR="${HOME}/.config/fish/conf.d/" - RC="${CONFIG_DIR}/python-${PYVER}.fish" - mkdir -p "$CONFIG_DIR" - if [ -f "${RC}" ]; then - cp -fp "${RC}" "${RC}.pysave" - fi - echo "# Setting PATH for Python ${PYVER}" > "${RC}" - if [ -f "${RC}.pysave" ]; then - echo "# The original version is saved in ${RC}.pysave" >> "${RC}" - fi - echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" - if [ `id -ur` = 0 ]; then - chown "${USER}" "${RC}" - fi - exit 0 - ;; + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" + + # Drop privileges while writing files. + su -m ${USER} < "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" +EOFF + + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; zsh) - PR="${HOME}/.zprofile" - ;; + PR="${HOME}/.zprofile" + ;; *sh) - PR="${HOME}/.profile" - ;; + PR="${HOME}/.profile" + ;; esac +# Drop privileges while writing files. +su -m ${USER} <> "${PR}" echo "# Setting PATH for Python ${PYVER}" >> "${PR}" echo "# The original version is saved in `basename ${PR}`.pysave" >> "${PR}" -echo 'PATH="'"${PYTHON_ROOT}/bin"':${PATH}"' >> "${PR}" +echo 'PATH="'"${PYTHON_ROOT}/bin"':\${PATH}"' >> "${PR}" echo 'export PATH' >> "${PR}" +EOFS + if [ `id -ur` = 0 ]; then - chown "${USER}" "${PR}" + chown -h "${USER}" "${PR}" fi exit 0 diff --git a/Makefile.pre.in b/Makefile.pre.in index b7b16ef4cb9d19..bcf19654adfb35 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -227,7 +227,6 @@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a -LIBHASHLIB_INTERNAL_A=Modules/_hashlib/libhashlib.a # HACL* build configuration LIBHACL_CFLAGS=@LIBHACL_CFLAGS@ @@ -762,17 +761,6 @@ LIBHACL_HMAC_HEADERS= \ $(LIBHACL_BLAKE2_HEADERS) \ $(LIBHACL_HEADERS) -########################################################################## -# Internal library for cryptographic primitives - -LIBHASHLIB_INTERNAL_OBJS= \ - Modules/_hashlib/hashlib_buffer.o - -LIBHASHLIB_INTERNAL_HEADERS= \ - Modules/_hashlib/hashlib_buffer.h \ - Modules/_hashlib/hashlib_fetch.h \ - Modules/_hashlib/hashlib_mutex.h - ######################################################################### # Rules @@ -1121,6 +1109,10 @@ web_example/index.html: $(WEBEX_DIR)/index.html @mkdir -p web_example @cp $< $@ +web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs + @mkdir -p web_example + @cp $< $@ + web_example/server.py: $(WEBEX_DIR)/server.py @mkdir -p web_example @cp $< $@ @@ -1138,7 +1130,7 @@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) cp python.wasm web_example/python.wasm .PHONY: web_example -web_example: web_example/python.mjs web_example/index.html web_example/server.py web_example/$(ZIP_STDLIB) +web_example: web_example/python.mjs web_example/python.worker.mjs web_example/index.html web_example/server.py web_example/$(ZIP_STDLIB) WEBEX2=web_example_pyrepl_jspi WEBEX2_DIR=$(EMSCRIPTEN_DIR)/$(WEBEX2)/ @@ -1523,17 +1515,6 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) -########################################################################## -# '_hashlib', '_hmac' and HACL*-based modules helpers -LIBHASHLIB_INTERNAL_CFLAGS=@LIBHASHLIB_INTERNAL_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) - -Modules/_hashlib/hashlib_buffer.o: Modules/_hashlib/hashlib_buffer.c $(LIBHASHLIB_INTERNAL_HEADERS) $(PYTHON_HEADERS) - $(CC) -I$(srcdir)/Modules/_hashlib -c $(LIBHASHLIB_INTERNAL_CFLAGS) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c - -$(LIBHASHLIB_INTERNAL_A): $(LIBHASHLIB_INTERNAL_OBJS) - -rm -f $@ - $(AR) $(ARFLAGS) $@ $(LIBHASHLIB_INTERNAL_OBJS) - ########################################################################## # HACL* library build # @@ -2249,7 +2230,7 @@ Python/frozen.o: $(FROZEN_FILES_OUT) # an include guard, so we can't use a pipeline to transform its output. Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d $(MKDIR_P) Include - CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -h -s $< + CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -h -s $(srcdir)/Include/pydtrace.d : sed in-place edit with POSIX-only tools sed 's/PYTHON_/PyDTrace_/' $@ > $@.tmp mv $@.tmp $@ @@ -2259,7 +2240,7 @@ Python/gc.o: $(srcdir)/Include/pydtrace.h Python/import.o: $(srcdir)/Include/pydtrace.h Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) - CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) + CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -G -s $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) Objects/typeobject.o: Objects/typeslots.inc @@ -3376,21 +3357,21 @@ MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ -MODULE__HASHLIB_DEPS=@LIBHASHLIB_INTERNAL@ +MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h # HACL*-based cryptographic primitives -MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA1_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA3_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__BLAKE2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__HMAC_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c diff --git a/Misc/ACKS b/Misc/ACKS index 745f472474cd9d..dc28ccf8f57eda 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1902,6 +1902,7 @@ Nicolas M. Thiéry James Thomas Reuben Thomas Robin Thomas +Douglas Thor Brian Thorne Christopher Thorne Stephen Thorne diff --git a/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst b/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst deleted file mode 100644 index 304f2c30f664db..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove :file:`!Modules/hashlib.h` and move its content into dedicated files -now located in ``Modules/_hashlib``. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/C_API/2025-07-31-04-30-42.gh-issue-128813.opL-Pv.rst b/Misc/NEWS.d/next/C_API/2025-07-31-04-30-42.gh-issue-128813.opL-Pv.rst new file mode 100644 index 00000000000000..caa8f3e9c985cd --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-07-31-04-30-42.gh-issue-128813.opL-Pv.rst @@ -0,0 +1,5 @@ +Functions :c:func:`_Py_c_sum`, :c:func:`_Py_c_diff`, :c:func:`_Py_c_neg`, +:c:func:`_Py_c_prod`, :c:func:`_Py_c_quot`, :c:func:`_Py_c_pow` and previously +undocumented :c:func:`_Py_c_abs` are :term:`soft deprecated`. Deprecate also +:c:member:`~PyComplexObject.cval` field of the :c:type:`PyComplexObject` type. +Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-10-50-46.gh-issue-134170.J0Hvmi.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-10-50-46.gh-issue-134170.J0Hvmi.rst new file mode 100644 index 00000000000000..f33a30c7e120dc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-10-50-46.gh-issue-134170.J0Hvmi.rst @@ -0,0 +1 @@ +Add colorization to :func:`sys.unraisablehook` by default. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-03-06-04-42.gh-issue-135552.CbBQof.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-03-06-04-42.gh-issue-135552.CbBQof.rst new file mode 100644 index 00000000000000..ea30a43fc25d41 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-03-06-04-42.gh-issue-135552.CbBQof.rst @@ -0,0 +1,7 @@ +Fix a bug caused by the garbage collector clearing weakrefs too early. The +weakrefs in the ``tp_subclasses`` dictionary are needed in order to correctly +invalidate type caches (for example, by calling ``PyType_Modified()``). +Clearing weakrefs before calling finalizers causes the caches to not be +correctly invalidated. That can cause crashes since the caches can refer to +invalid objects. Defer the clearing of weakrefs without callbacks until after +finalizers are executed. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-12-29-09.gh-issue-107545.ipfl7U.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-12-29-09.gh-issue-107545.ipfl7U.rst new file mode 100644 index 00000000000000..23122415e8a46f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-12-29-09.gh-issue-107545.ipfl7U.rst @@ -0,0 +1,2 @@ +Improve the error messages that may be raised by +:meth:`~socket.socket.setsockopt`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-28-19-11-34.gh-issue-134291.IiB9Id.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-28-19-11-34.gh-issue-134291.IiB9Id.rst new file mode 100644 index 00000000000000..1605bcc3574148 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-28-19-11-34.gh-issue-134291.IiB9Id.rst @@ -0,0 +1,2 @@ +Remove some newer macOS API usage from the JIT compiler in order to restore +compatibility with older OSX 10.15 deployment targets. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-31-23-02-02.gh-issue-137291.kIxVZd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-31-23-02-02.gh-issue-137291.kIxVZd.rst new file mode 100644 index 00000000000000..0995e3b4644e7c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-31-23-02-02.gh-issue-137291.kIxVZd.rst @@ -0,0 +1 @@ +The perf profiler can now be used if a previous frame evaluation API has been provided. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-01-18-54-31.gh-issue-137288.FhE7ku.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-01-18-54-31.gh-issue-137288.FhE7ku.rst new file mode 100644 index 00000000000000..37c143f18e76f7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-01-18-54-31.gh-issue-137288.FhE7ku.rst @@ -0,0 +1,2 @@ +Fix bug where some bytecode instructions of a boolean expression are not +associated with the correct exception handler. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-10-27-53.gh-issue-137308.at05p_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-10-27-53.gh-issue-137308.at05p_.rst new file mode 100644 index 00000000000000..8003de422b2919 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-10-27-53.gh-issue-137308.at05p_.rst @@ -0,0 +1,3 @@ +A standalone docstring in a node body is optimized as a :keyword:`pass` +statement to ensure that the node's body is never empty. There was a +:exc:`ValueError` in :func:`compile` otherwise. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-23-04-57.gh-issue-137314.wjEdzD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-23-04-57.gh-issue-137314.wjEdzD.rst new file mode 100644 index 00000000000000..09d0c3e68fc1ed --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-02-23-04-57.gh-issue-137314.wjEdzD.rst @@ -0,0 +1,5 @@ +Fixed a regression where raw f-strings incorrectly interpreted +escape sequences in format specifications. Raw f-strings now properly preserve +literal backslashes in format specs, matching the behavior from Python 3.11. +For example, ``rf"{obj:\xFF}"`` now correctly produces ``'\\xFF'`` instead of +``'ÿ'``. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-05-17-22-24.gh-issue-58124.q1__53.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-05-17-22-24.gh-issue-58124.q1__53.rst new file mode 100644 index 00000000000000..f875d4c5e785c6 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-05-17-22-24.gh-issue-58124.q1__53.rst @@ -0,0 +1,3 @@ +Fix name of the Python encoding in Unicode errors of the code page codec: +use "cp65000" and "cp65001" instead of "CP_UTF7" and "CP_UTF8" which are not +valid Python code names. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst b/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst deleted file mode 100644 index 0341b5f7f0d5e6..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst +++ /dev/null @@ -1 +0,0 @@ -EPUB builds are fixed by excluding non-XHTML-compatible tags. diff --git a/Misc/NEWS.d/next/Library/2021-09-21-17-17-29.gh-issue-84683.wDSRsG.rst b/Misc/NEWS.d/next/Library/2021-09-21-17-17-29.gh-issue-84683.wDSRsG.rst new file mode 100644 index 00000000000000..66f76bda6ad7a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-09-21-17-17-29.gh-issue-84683.wDSRsG.rst @@ -0,0 +1 @@ +:mod:`zoneinfo`: Check in ``/share/zoneinfo`` for data files on Windows diff --git a/Misc/NEWS.d/next/Library/2025-03-17-21-21-06.gh-issue-131146.A5Obgv.rst b/Misc/NEWS.d/next/Library/2025-03-17-21-21-06.gh-issue-131146.A5Obgv.rst new file mode 100644 index 00000000000000..6d8bc0959aed52 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-17-21-21-06.gh-issue-131146.A5Obgv.rst @@ -0,0 +1,6 @@ +Fix :class:`calendar.TextCalendar`, :class:`calendar.HTMLCalendar`, +and the :mod:`calendar` CLI to display month names in the nominative +case by adding :data:`calendar.standalone_month_name` and +:data:`calendar.standalone_month_abbr`, which provide month names and +abbreviations in the grammatical form used when a month name stands by +itself, if the locale supports it. diff --git a/Misc/NEWS.d/next/Library/2025-05-08-20-03-20.gh-issue-133722.1-B82a.rst b/Misc/NEWS.d/next/Library/2025-05-08-20-03-20.gh-issue-133722.1-B82a.rst new file mode 100644 index 00000000000000..86f244412498c4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-08-20-03-20.gh-issue-133722.1-B82a.rst @@ -0,0 +1,2 @@ +Added a *color* option to :func:`difflib.unified_diff` that colors output +similar to :program:`git diff`. diff --git a/Misc/NEWS.d/next/Library/2025-05-29-19-00-37.gh-issue-134861.y2-fu-.rst b/Misc/NEWS.d/next/Library/2025-05-29-19-00-37.gh-issue-134861.y2-fu-.rst new file mode 100644 index 00000000000000..07e4c61b404ba4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-29-19-00-37.gh-issue-134861.y2-fu-.rst @@ -0,0 +1 @@ +Add CSV as an output format for :program:`python -m asyncio ps`. diff --git a/Misc/NEWS.d/next/Library/2025-06-11-15-08-02.gh-issue-135336.6Gq6MI.rst b/Misc/NEWS.d/next/Library/2025-06-11-15-08-02.gh-issue-135336.6Gq6MI.rst new file mode 100644 index 00000000000000..8a1d492ff08944 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-11-15-08-02.gh-issue-135336.6Gq6MI.rst @@ -0,0 +1 @@ +:mod:`json` now encodes strings up to 2.2x faster if they consist solely of characters that don’t require escaping. diff --git a/Misc/NEWS.d/next/Library/2025-07-12-14-15-47.gh-issue-136571.muHmBv.rst b/Misc/NEWS.d/next/Library/2025-07-12-14-15-47.gh-issue-136571.muHmBv.rst new file mode 100644 index 00000000000000..37f535f564883c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-12-14-15-47.gh-issue-136571.muHmBv.rst @@ -0,0 +1,2 @@ +:meth:`datetime.date.fromisocalendar` can now raise OverflowError for out of +range arguments. diff --git a/Misc/NEWS.d/next/Library/2025-07-28-23-11-29.gh-issue-81325.jMJFBe.rst b/Misc/NEWS.d/next/Library/2025-07-28-23-11-29.gh-issue-81325.jMJFBe.rst new file mode 100644 index 00000000000000..3d89b6eb92a0d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-28-23-11-29.gh-issue-81325.jMJFBe.rst @@ -0,0 +1,2 @@ +:class:`tarfile.TarFile` now accepts a :term:`path-like ` when working on a tar archive. +(Contributed by Alexander Enrique Urieles Nieto in :gh:`81325`.) diff --git a/Misc/NEWS.d/next/Library/2025-07-30-11-12-22.gh-issue-124503.d4hc7b.rst b/Misc/NEWS.d/next/Library/2025-07-30-11-12-22.gh-issue-124503.d4hc7b.rst new file mode 100644 index 00000000000000..c04eba932a0f2e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-30-11-12-22.gh-issue-124503.d4hc7b.rst @@ -0,0 +1 @@ +:func:`ast.literal_eval` is 10-20% faster for small inputs. diff --git a/Misc/NEWS.d/next/Library/2025-07-30-18-07-33.gh-issue-137257.XBtzf2.rst b/Misc/NEWS.d/next/Library/2025-07-30-18-07-33.gh-issue-137257.XBtzf2.rst new file mode 100644 index 00000000000000..fad609854025c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-30-18-07-33.gh-issue-137257.XBtzf2.rst @@ -0,0 +1 @@ +Bump the version of pip bundled in ensurepip to version 25.2 diff --git a/Misc/NEWS.d/next/Library/2025-07-31-10-31-56.gh-issue-137282.GOCwIC.rst b/Misc/NEWS.d/next/Library/2025-07-31-10-31-56.gh-issue-137282.GOCwIC.rst new file mode 100644 index 00000000000000..78f169ea029b17 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-31-10-31-56.gh-issue-137282.GOCwIC.rst @@ -0,0 +1 @@ +Fix tab completion and :func:`dir` on :mod:`concurrent.futures`. diff --git a/Misc/NEWS.d/next/Library/2025-07-31-16-43-16.gh-issue-137191.FIogE8.rst b/Misc/NEWS.d/next/Library/2025-07-31-16-43-16.gh-issue-137191.FIogE8.rst new file mode 100644 index 00000000000000..b2dba81251eed6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-31-16-43-16.gh-issue-137191.FIogE8.rst @@ -0,0 +1,4 @@ +Fix how type parameters are collected, when :class:`typing.Protocol` are +specified with explicit parameters. Now, :class:`typing.Generic` and +:class:`typing.Protocol` always dictate the parameter number +and parameter ordering of types. Previous behavior was a bug. diff --git a/Misc/NEWS.d/next/Library/2025-08-01-15-07-59.gh-issue-137273.4V8Xmv.rst b/Misc/NEWS.d/next/Library/2025-08-01-15-07-59.gh-issue-137273.4V8Xmv.rst new file mode 100644 index 00000000000000..f344877955fea0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-01-15-07-59.gh-issue-137273.4V8Xmv.rst @@ -0,0 +1 @@ +Fix debug assertion failure in :func:`locale.setlocale` on Windows. diff --git a/Misc/NEWS.d/next/Library/2025-08-01-23-52-49.gh-issue-75989.5aYXNJ.rst b/Misc/NEWS.d/next/Library/2025-08-01-23-52-49.gh-issue-75989.5aYXNJ.rst new file mode 100644 index 00000000000000..00b15503b50ba3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-01-23-52-49.gh-issue-75989.5aYXNJ.rst @@ -0,0 +1,3 @@ +:func:`tarfile.TarFile.extractall` and :func:`tarfile.TarFile.extract` now +overwrite symlinks when extracting hardlinks. +(Contributed by Alexander Enrique Urieles Nieto in :gh:`75989`.) diff --git a/Misc/NEWS.d/next/Library/2025-08-03-13-16-39.gh-issue-137044.0hPVL_.rst b/Misc/NEWS.d/next/Library/2025-08-03-13-16-39.gh-issue-137044.0hPVL_.rst new file mode 100644 index 00000000000000..f5f96263823e86 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-03-13-16-39.gh-issue-137044.0hPVL_.rst @@ -0,0 +1,4 @@ +Return large limit values as positive integers instead of negative integers +in :func:`resource.getrlimit`. Accept large values and reject negative +values (except :data:`~resource.RLIM_INFINITY`) for limits in +:func:`resource.setrlimit`. diff --git a/Misc/NEWS.d/next/Library/2025-08-06-16-13-47.gh-issue-137466.Whv0-A.rst b/Misc/NEWS.d/next/Library/2025-08-06-16-13-47.gh-issue-137466.Whv0-A.rst new file mode 100644 index 00000000000000..918019aa8c03eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-06-16-13-47.gh-issue-137466.Whv0-A.rst @@ -0,0 +1,3 @@ +Remove undocumented :func:`!glob.glob0` and :func:`!glob.glob1` functions, +which have been deprecated since Python 3.13. Use :func:`glob.glob` and pass +a directory to its *root_dir* argument instead. diff --git a/Misc/NEWS.d/next/Library/2025-08-08-21-20-14.gh-issue-92936.rOgG1S.rst b/Misc/NEWS.d/next/Library/2025-08-08-21-20-14.gh-issue-92936.rOgG1S.rst new file mode 100644 index 00000000000000..906c442b64f438 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-08-21-20-14.gh-issue-92936.rOgG1S.rst @@ -0,0 +1,2 @@ +Update regex used by ``http.cookies.SimpleCookie`` to handle values containing +double quotes. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-08-06-11-54-55.gh-issue-137484.8iFAQs.rst b/Misc/NEWS.d/next/Tools-Demos/2025-08-06-11-54-55.gh-issue-137484.8iFAQs.rst new file mode 100644 index 00000000000000..bd7bc0984ecfcc --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-08-06-11-54-55.gh-issue-137484.8iFAQs.rst @@ -0,0 +1,2 @@ +Have ``Tools/wasm/wasi`` put the build Python into a directory named after +the build triple instead of "build". diff --git a/Misc/NEWS.d/next/Windows/2025-07-27-02-16-53.gh-issue-137134.W0WpDF.rst b/Misc/NEWS.d/next/Windows/2025-07-27-02-16-53.gh-issue-137134.W0WpDF.rst new file mode 100644 index 00000000000000..ddccf95b7d039a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-07-27-02-16-53.gh-issue-137134.W0WpDF.rst @@ -0,0 +1 @@ +Update Windows installer to ship with SQLite 3.50.4. diff --git a/Misc/NEWS.d/next/macOS/2025-07-27-02-17-40.gh-issue-137134.pjgITs.rst b/Misc/NEWS.d/next/macOS/2025-07-27-02-17-40.gh-issue-137134.pjgITs.rst new file mode 100644 index 00000000000000..957270f5abae93 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2025-07-27-02-17-40.gh-issue-137134.pjgITs.rst @@ -0,0 +1 @@ +Update macOS installer to ship with SQLite version 3.50.4. diff --git a/Misc/NEWS.d/next/macOS/2025-08-06-06-29-12.gh-issue-137450.JZypb7.rst b/Misc/NEWS.d/next/macOS/2025-08-06-06-29-12.gh-issue-137450.JZypb7.rst new file mode 100644 index 00000000000000..5efd74660c95d2 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2025-08-06-06-29-12.gh-issue-137450.JZypb7.rst @@ -0,0 +1,3 @@ +macOS installer shell path management improvements: separate the installer +``Shell profile updater`` postinstall script from the +``Update Shell Profile.command`` to enable more robust error handling. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 69f3beec82ed34..a87af7f9173780 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -91,21 +91,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "e335aeb44fa36cde60ecbb6a9f8be6f5d449d645ce9b0199ee53a7e6728d19d2" + "checksumValue": "fb5ab81f27612b0a7b4861ba655906c76dc85ee969e7a4905d2075aff931e8d0" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.49.1.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.50.4.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.49.1.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.50.4.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "sqlite", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.49.1.0" + "versionInfo": "3.50.4.0" }, { "SPDXID": "SPDXRef-PACKAGE-tcl-core", diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index 80ba96614bff79..db405acf8727b5 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -30,11 +30,6 @@ #ifdef Py_GIL_DISABLED static PyMutex malloc_closure_lock; -# define MALLOC_CLOSURE_LOCK() PyMutex_Lock(&malloc_closure_lock) -# define MALLOC_CLOSURE_UNLOCK() PyMutex_Unlock(&malloc_closure_lock) -#else -# define MALLOC_CLOSURE_LOCK() ((void)0) -# define MALLOC_CLOSURE_UNLOCK() ((void)0) #endif typedef union _tagITEM { @@ -120,11 +115,11 @@ void Py_ffi_closure_free(void *p) } #endif #endif - MALLOC_CLOSURE_LOCK(); + FT_MUTEX_LOCK(&malloc_closure_lock); ITEM *item = (ITEM *)p; item->next = free_list; free_list = item; - MALLOC_CLOSURE_UNLOCK(); + FT_MUTEX_UNLOCK(&malloc_closure_lock); } /* return one item from the free list, allocating more if needed */ @@ -143,13 +138,13 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc) } #endif #endif - MALLOC_CLOSURE_LOCK(); + FT_MUTEX_LOCK(&malloc_closure_lock); ITEM *item; if (!free_list) { more_core(); } if (!free_list) { - MALLOC_CLOSURE_UNLOCK(); + FT_MUTEX_UNLOCK(&malloc_closure_lock); return NULL; } item = free_list; @@ -160,6 +155,6 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc) #else *codeloc = (void *)item; #endif - MALLOC_CLOSURE_UNLOCK(); + FT_MUTEX_UNLOCK(&malloc_closure_lock); return (void *)item; } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 01039dfeec0719..e0bd4993817277 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -214,7 +214,7 @@ clear_current_module(PyInterpreterState *interp, PyObject *expected) if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { goto error; } - if (ref != NULL) { + if (ref != NULL && ref != Py_None) { PyObject *current = NULL; int rc = PyWeakref_GetRef(ref, ¤t); /* We only need "current" for pointer comparison. */ @@ -331,8 +331,10 @@ class datetime.datetime "PyDateTime_DateTime *" "get_datetime_state()->datetime_ class datetime.date "PyDateTime_Date *" "get_datetime_state()->date_type" class datetime.time "PyDateTime_Time *" "get_datetime_state()->time_type" class datetime.IsoCalendarDate "PyDateTime_IsoCalendarDate *" "get_datetime_state()->isocalendar_date_type" +class datetime.timedelta "PyDateTime_Delta *" "&PyDateTime_DeltaType" +class datetime.timezone "PyDateTime_TimeZone *" "&PyDateTime_TimeZoneType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c8f3d834a860d50a]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=c54b9adf60082f0d]*/ #include "clinic/_datetimemodule.c.h" @@ -1171,19 +1173,18 @@ new_datetime_ex(int, int, int, int, int, int, int, PyObject *, PyTypeObject *); /* Create date instance with no range checking, or call subclass constructor */ static PyObject * -new_date_subclass_ex(int year, int month, int day, PyObject *cls) +new_date_subclass_ex(int year, int month, int day, PyTypeObject *cls) { PyObject *result; // We have "fast path" constructors for two subclasses: date and datetime - if ((PyTypeObject *)cls == DATE_TYPE(NO_STATE)) { - result = new_date_ex(year, month, day, (PyTypeObject *)cls); + if (cls == DATE_TYPE(NO_STATE)) { + result = new_date_ex(year, month, day, cls); } - else if ((PyTypeObject *)cls == DATETIME_TYPE(NO_STATE)) { - result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, - (PyTypeObject *)cls); + else if (cls == DATETIME_TYPE(NO_STATE)) { + result = new_datetime_ex(year, month, day, 0, 0, 0, 0, Py_None, cls); } else { - result = PyObject_CallFunction(cls, "iii", year, month, day); + result = PyObject_CallFunction((PyObject *)cls, "iii", year, month, day); } return result; @@ -1235,7 +1236,7 @@ new_datetime_ex(int year, int month, int day, int hour, int minute, new_datetime_ex2(y, m, d, hh, mm, ss, us, tzinfo, fold, DATETIME_TYPE(NO_STATE)) static PyObject * -call_subclass_fold(PyObject *cls, int fold, const char *format, ...) +call_subclass_fold(PyTypeObject *cls, int fold, const char *format, ...) { PyObject *kwargs = NULL, *res = NULL; va_list va; @@ -1261,7 +1262,7 @@ call_subclass_fold(PyObject *cls, int fold, const char *format, ...) goto Done; } } - res = PyObject_Call(cls, args, kwargs); + res = PyObject_Call((PyObject *)cls, args, kwargs); Done: Py_DECREF(args); Py_XDECREF(kwargs); @@ -1271,10 +1272,10 @@ call_subclass_fold(PyObject *cls, int fold, const char *format, ...) static PyObject * new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, - int fold, PyObject *cls) + int fold, PyTypeObject *cls) { PyObject* dt; - if ((PyTypeObject*)cls == DATETIME_TYPE(NO_STATE)) { + if (cls == DATETIME_TYPE(NO_STATE)) { // Use the fast path constructor dt = new_datetime(year, month, day, hour, minute, second, usecond, tzinfo, fold); @@ -1291,7 +1292,7 @@ new_datetime_subclass_fold_ex(int year, int month, int day, int hour, int minute static PyObject * new_datetime_subclass_ex(int year, int month, int day, int hour, int minute, int second, int usecond, PyObject *tzinfo, - PyObject *cls) { + PyTypeObject *cls) { return new_datetime_subclass_fold_ex(year, month, day, hour, minute, second, usecond, tzinfo, 0, cls); @@ -1340,10 +1341,10 @@ new_time_ex(int hour, int minute, int second, int usecond, static PyObject * new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, - PyObject *tzinfo, int fold, PyObject *cls) + PyObject *tzinfo, int fold, PyTypeObject *cls) { PyObject *t; - if ((PyTypeObject*)cls == TIME_TYPE(NO_STATE)) { + if (cls == TIME_TYPE(NO_STATE)) { // Use the fast path constructor t = new_time(hour, minute, second, usecond, tzinfo, fold); } @@ -2768,38 +2769,39 @@ accum(const char* tag, PyObject *sofar, PyObject *num, PyObject *factor, return NULL; } +/*[clinic input] +@classmethod +datetime.timedelta.__new__ as delta_new + + days: object(c_default="NULL") = 0 + seconds: object(c_default="NULL") = 0 + microseconds: object(c_default="NULL") = 0 + milliseconds: object(c_default="NULL") = 0 + minutes: object(c_default="NULL") = 0 + hours: object(c_default="NULL") = 0 + weeks: object(c_default="NULL") = 0 + +Difference between two datetime values. + +All arguments are optional and default to 0. +Arguments may be integers or floats, and may be positive or negative. +[clinic start generated code]*/ + static PyObject * -delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) +delta_new_impl(PyTypeObject *type, PyObject *days, PyObject *seconds, + PyObject *microseconds, PyObject *milliseconds, + PyObject *minutes, PyObject *hours, PyObject *weeks) +/*[clinic end generated code: output=61d7e02a92a97700 input=e8cd54819295d34b]*/ { PyObject *self = NULL; PyObject *current_mod = NULL; datetime_state *st = GET_CURRENT_STATE(current_mod); - /* Argument objects. */ - PyObject *day = NULL; - PyObject *second = NULL; - PyObject *us = NULL; - PyObject *ms = NULL; - PyObject *minute = NULL; - PyObject *hour = NULL; - PyObject *week = NULL; - PyObject *x = NULL; /* running sum of microseconds */ PyObject *y = NULL; /* temp sum of microseconds */ double leftover_us = 0.0; - static char *keywords[] = { - "days", "seconds", "microseconds", "milliseconds", - "minutes", "hours", "weeks", NULL - }; - - if (PyArg_ParseTupleAndKeywords(args, kw, "|OOOOOOO:__new__", - keywords, - &day, &second, &us, - &ms, &minute, &hour, &week) == 0) - goto Done; - x = PyLong_FromLong(0); if (x == NULL) goto Done; @@ -2810,32 +2812,32 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (x == NULL) \ goto Done - if (us) { - y = accum("microseconds", x, us, _PyLong_GetOne(), &leftover_us); + if (microseconds) { + y = accum("microseconds", x, microseconds, _PyLong_GetOne(), &leftover_us); CLEANUP; } - if (ms) { - y = accum("milliseconds", x, ms, CONST_US_PER_MS(st), &leftover_us); + if (milliseconds) { + y = accum("milliseconds", x, milliseconds, CONST_US_PER_MS(st), &leftover_us); CLEANUP; } - if (second) { - y = accum("seconds", x, second, CONST_US_PER_SECOND(st), &leftover_us); + if (seconds) { + y = accum("seconds", x, seconds, CONST_US_PER_SECOND(st), &leftover_us); CLEANUP; } - if (minute) { - y = accum("minutes", x, minute, CONST_US_PER_MINUTE(st), &leftover_us); + if (minutes) { + y = accum("minutes", x, minutes, CONST_US_PER_MINUTE(st), &leftover_us); CLEANUP; } - if (hour) { - y = accum("hours", x, hour, CONST_US_PER_HOUR(st), &leftover_us); + if (hours) { + y = accum("hours", x, hours, CONST_US_PER_HOUR(st), &leftover_us); CLEANUP; } - if (day) { - y = accum("days", x, day, CONST_US_PER_DAY(st), &leftover_us); + if (days) { + y = accum("days", x, days, CONST_US_PER_DAY(st), &leftover_us); CLEANUP; } - if (week) { - y = accum("weeks", x, week, CONST_US_PER_WEEK(st), &leftover_us); + if (weeks) { + y = accum("weeks", x, weeks, CONST_US_PER_WEEK(st), &leftover_us); CLEANUP; } if (leftover_us) { @@ -3034,13 +3036,6 @@ static PyMethodDef delta_methods[] = { {NULL, NULL}, }; -static const char delta_doc[] = -PyDoc_STR("Difference between two datetime values.\n\n" - "timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, " - "minutes=0, hours=0, weeks=0)\n\n" - "All arguments are optional and default to 0.\n" - "Arguments may be integers or floats, and may be positive or negative."); - static PyNumberMethods delta_as_number = { delta_add, /* nb_add */ delta_subtract, /* nb_subtract */ @@ -3098,7 +3093,7 @@ static PyTypeObject PyDateTime_DeltaType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - delta_doc, /* tp_doc */ + delta_new__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ delta_richcompare, /* tp_richcompare */ @@ -3174,8 +3169,6 @@ static PyGetSetDef date_getset[] = { /* Constructors. */ -static char *date_kws[] = {"year", "month", "day", NULL}; - static PyObject * date_from_pickle(PyTypeObject *type, PyObject *state) { @@ -3193,11 +3186,6 @@ date_from_pickle(PyTypeObject *type, PyObject *state) static PyObject * date_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *self = NULL; - int year; - int month; - int day; - /* Check for invocation from pickle with __getstate__ state */ if (PyTuple_GET_SIZE(args) == 1) { PyObject *state = PyTuple_GET_ITEM(args, 0); @@ -3223,22 +3211,36 @@ date_new(PyTypeObject *type, PyObject *args, PyObject *kw) } return NULL; } - self = date_from_pickle(type, state); + PyObject *self = date_from_pickle(type, state); Py_DECREF(state); return self; } } } - if (PyArg_ParseTupleAndKeywords(args, kw, "iii", date_kws, - &year, &month, &day)) { - self = new_date_ex(year, month, day, type); - } - return self; + return datetime_date(type, args, kw); +} + +/*[clinic input] +@classmethod +datetime.date.__new__ + + year: int + month: int + day: int + +Concrete date type. +[clinic start generated code]*/ + +static PyObject * +datetime_date_impl(PyTypeObject *type, int year, int month, int day) +/*[clinic end generated code: output=6654caa3dea7d518 input=fd1bac0658690455]*/ +{ + return new_date_ex(year, month, day, type); } static PyObject * -date_fromtimestamp(PyObject *cls, PyObject *obj) +date_fromtimestamp(PyTypeObject *cls, PyObject *obj) { struct tm tm; time_t t; @@ -3260,8 +3262,18 @@ date_fromtimestamp(PyObject *cls, PyObject *obj) * only way to be sure of that is to *call* time.time(). That's not * generally the same as calling C's time. */ +/*[clinic input] +@classmethod +datetime.date.today + +Current date or datetime. + +Equivalent to fromtimestamp(time.time()). +[clinic start generated code]*/ + static PyObject * -date_today(PyObject *cls, PyObject *Py_UNUSED(dummy)) +datetime_date_today_impl(PyTypeObject *type) +/*[clinic end generated code: output=d5474697df6b251c input=21688afa289c0a06]*/ { PyObject *time; PyObject *result; @@ -3275,7 +3287,7 @@ date_today(PyObject *cls, PyObject *Py_UNUSED(dummy)) * time.time() delivers; if someone were gonzo about optimization, * date.today() could get away with plain C time(). */ - result = PyObject_CallMethodOneArg(cls, &_Py_ID(fromtimestamp), time); + result = PyObject_CallMethodOneArg((PyObject*)type, &_Py_ID(fromtimestamp), time); Py_DECREF(time); return result; } @@ -3297,7 +3309,7 @@ static PyObject * datetime_date_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp) /*[clinic end generated code: output=59def4e32c028fb6 input=eabb3fe7f40491fe]*/ { - return date_fromtimestamp((PyObject *) type, timestamp); + return date_fromtimestamp(type, timestamp); } /* bpo-36025: This is a wrapper for API compatibility with the public C API, @@ -3311,52 +3323,58 @@ datetime_date_fromtimestamp_capi(PyObject *cls, PyObject *args) PyObject *result = NULL; if (PyArg_UnpackTuple(args, "fromtimestamp", 1, 1, ×tamp)) { - result = date_fromtimestamp(cls, timestamp); + result = date_fromtimestamp((PyTypeObject *)cls, timestamp); } return result; } -/* Return new date from proleptic Gregorian ordinal. Raises ValueError if - * the ordinal is out of range. - */ -static PyObject * -date_fromordinal(PyObject *cls, PyObject *args) -{ - PyObject *result = NULL; - int ordinal; +/*[clinic input] +@classmethod +datetime.date.fromordinal - if (PyArg_ParseTuple(args, "i:fromordinal", &ordinal)) { - int year; - int month; - int day; + ordinal: int + / - if (ordinal < 1) - PyErr_SetString(PyExc_ValueError, "ordinal must be " - ">= 1"); - else { - ord_to_ymd(ordinal, &year, &month, &day); - result = new_date_subclass_ex(year, month, day, cls); - } - } - return result; -} +Construct a date from a proleptic Gregorian ordinal. + +January 1 of year 1 is day 1. Only the year, month and day are +non-zero in the result. +[clinic start generated code]*/ -/* Return the new date from a string as generated by date.isoformat() */ static PyObject * -date_fromisoformat(PyObject *cls, PyObject *dtstr) +datetime_date_fromordinal_impl(PyTypeObject *type, int ordinal) +/*[clinic end generated code: output=ea5cc69d86614a6b input=a3a4eedf582f145e]*/ { - assert(dtstr != NULL); + int year; + int month; + int day; - if (!PyUnicode_Check(dtstr)) { - PyErr_SetString(PyExc_TypeError, - "fromisoformat: argument must be str"); + if (ordinal < 1) { + PyErr_SetString(PyExc_ValueError, "ordinal must be >= 1"); return NULL; } + ord_to_ymd(ordinal, &year, &month, &day); + return new_date_subclass_ex(year, month, day, type); +} + +/*[clinic input] +@classmethod +datetime.date.fromisoformat + string: unicode + / + +Construct a date from a string in ISO 8601 format. +[clinic start generated code]*/ + +static PyObject * +datetime_date_fromisoformat_impl(PyTypeObject *type, PyObject *string) +/*[clinic end generated code: output=8b9f9324904fca02 input=73c64216c10bcc8e]*/ +{ Py_ssize_t len; - const char *dt_ptr = PyUnicode_AsUTF8AndSize(dtstr, &len); + const char *dt_ptr = PyUnicode_AsUTF8AndSize(string, &len); if (dt_ptr == NULL) { goto invalid_string_error; } @@ -3375,33 +3393,32 @@ date_fromisoformat(PyObject *cls, PyObject *dtstr) goto invalid_string_error; } - return new_date_subclass_ex(year, month, day, cls); + return new_date_subclass_ex(year, month, day, type); invalid_string_error: - PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", dtstr); + PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", string); return NULL; } -static PyObject * -date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) -{ - static char *keywords[] = { - "year", "week", "day", NULL - }; +/*[clinic input] +@classmethod +datetime.date.fromisocalendar - int year, week, day; - if (PyArg_ParseTupleAndKeywords(args, kw, "iii:fromisocalendar", - keywords, - &year, &week, &day) == 0) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - PyErr_Format(PyExc_ValueError, - "ISO calendar component out of range"); + year: int + week: int + day: int - } - return NULL; - } +Construct a date from the ISO year, week number and weekday. + +This is the inverse of the date.isocalendar() function. +[clinic start generated code]*/ +static PyObject * +datetime_date_fromisocalendar_impl(PyTypeObject *type, int year, int week, + int day) +/*[clinic end generated code: output=7b26e15115d24df6 input=fbb05b53d6fb51d8]*/ +{ int month; int rv = iso_to_ymd(year, week, day, &year, &month, &day); @@ -3422,26 +3439,34 @@ date_fromisocalendar(PyObject *cls, PyObject *args, PyObject *kw) return NULL; } - return new_date_subclass_ex(year, month, day, cls); + return new_date_subclass_ex(year, month, day, type); } -/* Return new date from _strptime.strptime_datetime_date(). */ +/*[clinic input] +@classmethod +datetime.date.strptime + + string: unicode + format: unicode + / + +Parse string according to the given date format (like time.strptime()). +[clinic start generated code]*/ + static PyObject * -date_strptime(PyObject *cls, PyObject *args) +datetime_date_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format) +/*[clinic end generated code: output=454d473bee2d5161 input=001904ab34f594a1]*/ { - PyObject *string, *format, *result; - - if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { - return NULL; - } + PyObject *result; PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { return NULL; } result = PyObject_CallMethodObjArgs(module, - &_Py_ID(_strptime_datetime_date), cls, - string, format, NULL); + &_Py_ID(_strptime_datetime_date), + (PyObject *)type, string, format, NULL); Py_DECREF(module); return result; } @@ -3465,8 +3490,7 @@ add_date_timedelta(PyDateTime_Date *date, PyDateTime_Delta *delta, int negate) int day = GET_DAY(date) + (negate ? -deltadays : deltadays); if (normalize_date(&year, &month, &day) >= 0) - result = new_date_subclass_ex(year, month, day, - (PyObject* )Py_TYPE(date)); + result = new_date_subclass_ex(year, month, day, Py_TYPE(date)); return result; } @@ -3558,20 +3582,26 @@ date_ctime(PyObject *self, PyObject *Py_UNUSED(dummy)) return format_ctime(self, 0, 0, 0); } +/*[clinic input] +datetime.date.strftime + + self: self(type="PyObject *") + format: unicode + +Format using strftime(). + +Example: "%d/%m/%Y, %H:%M:%S". +[clinic start generated code]*/ + static PyObject * -date_strftime(PyObject *self, PyObject *args, PyObject *kw) +datetime_date_strftime_impl(PyObject *self, PyObject *format) +/*[clinic end generated code: output=6529b70095e16778 input=72af55077e606ed8]*/ { /* This method can be inherited, and needs to call the * timetuple() method appropriate to self's class. */ PyObject *result; PyObject *tuple; - PyObject *format; - static char *keywords[] = {"format", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kw, "U:strftime", keywords, - &format)) - return NULL; tuple = PyObject_CallMethodNoArgs(self, &_Py_ID(timetuple)); if (tuple == NULL) @@ -3581,14 +3611,20 @@ date_strftime(PyObject *self, PyObject *args, PyObject *kw) return result; } -static PyObject * -date_format(PyObject *self, PyObject *args) -{ - PyObject *format; +/*[clinic input] +datetime.date.__format__ - if (!PyArg_ParseTuple(args, "U:__format__", &format)) - return NULL; + self: self(type="PyObject *") + format: unicode + / + +Formats self with strftime. +[clinic start generated code]*/ +static PyObject * +datetime_date___format___impl(PyObject *self, PyObject *format) +/*[clinic end generated code: output=efa0223d000a93b7 input=e417a7c84e1abaf9]*/ +{ /* if the format is zero length, return str(self) */ if (PyUnicode_GetLength(format) == 0) return PyObject_Str(self); @@ -3834,7 +3870,7 @@ datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, int day) /*[clinic end generated code: output=2a9430d1e6318aeb input=0d1f02685b3e90f6]*/ { - return new_date_subclass_ex(year, month, day, (PyObject *)Py_TYPE(self)); + return new_date_subclass_ex(year, month, day, Py_TYPE(self)); } static Py_hash_t @@ -3895,38 +3931,19 @@ static PyMethodDef date_methods[] = { /* Class methods: */ DATETIME_DATE_FROMTIMESTAMP_METHODDEF - - {"fromordinal", date_fromordinal, METH_VARARGS | METH_CLASS, - PyDoc_STR("int -> date corresponding to a proleptic Gregorian " - "ordinal.")}, - - {"fromisoformat", date_fromisoformat, METH_O | METH_CLASS, - PyDoc_STR("str -> Construct a date from a string in ISO 8601 format.")}, - - {"fromisocalendar", _PyCFunction_CAST(date_fromisocalendar), - METH_VARARGS | METH_KEYWORDS | METH_CLASS, - PyDoc_STR("int, int, int -> Construct a date from the ISO year, week " - "number and weekday.\n\n" - "This is the inverse of the date.isocalendar() function")}, - - {"strptime", date_strptime, METH_VARARGS | METH_CLASS, - PyDoc_STR("string, format -> new date parsed from a string " - "(like time.strptime()).")}, - - {"today", date_today, METH_NOARGS | METH_CLASS, - PyDoc_STR("Current date or datetime: same as " - "self.__class__.fromtimestamp(time.time()).")}, + DATETIME_DATE_FROMORDINAL_METHODDEF + DATETIME_DATE_FROMISOFORMAT_METHODDEF + DATETIME_DATE_FROMISOCALENDAR_METHODDEF + DATETIME_DATE_STRPTIME_METHODDEF + DATETIME_DATE_TODAY_METHODDEF /* Instance methods: */ {"ctime", date_ctime, METH_NOARGS, PyDoc_STR("Return ctime() style string.")}, - {"strftime", _PyCFunction_CAST(date_strftime), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("format -> strftime() style string.")}, - - {"__format__", date_format, METH_VARARGS, - PyDoc_STR("Formats self with strftime.")}, + DATETIME_DATE_STRFTIME_METHODDEF + DATETIME_DATE___FORMAT___METHODDEF {"timetuple", date_timetuple, METH_NOARGS, PyDoc_STR("Return time tuple, compatible with time.localtime().")}, @@ -3961,9 +3978,6 @@ static PyMethodDef date_methods[] = { {NULL, NULL} }; -static const char date_doc[] = -PyDoc_STR("date(year, month, day) --> date object"); - static PyNumberMethods date_as_number = { date_add, /* nb_add */ date_subtract, /* nb_subtract */ @@ -3998,7 +4012,7 @@ static PyTypeObject PyDateTime_DateType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - date_doc, /* tp_doc */ + datetime_date__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ date_richcompare, /* tp_richcompare */ @@ -4201,7 +4215,8 @@ static PyMethodDef tzinfo_methods[] = { }; static const char tzinfo_doc[] = -PyDoc_STR("Abstract base class for time zone info objects."); +PyDoc_STR("Abstract base class for time zone info objects.\n\n" + "Subclasses must override the tzname(), utcoffset() and dst() methods."); static PyTypeObject PyDateTime_TZInfoType = { PyVarObject_HEAD_INIT(NULL, 0) @@ -4245,18 +4260,21 @@ static PyTypeObject PyDateTime_TZInfoType = { 0, /* tp_free */ }; -static char *timezone_kws[] = {"offset", "name", NULL}; +/*[clinic input] +@classmethod +datetime.timezone.__new__ as timezone_new + + offset: object(subclass_of="DELTA_TYPE(NO_STATE)") + name: unicode = NULL + +Fixed offset from UTC implementation of tzinfo. +[clinic start generated code]*/ static PyObject * -timezone_new(PyTypeObject *type, PyObject *args, PyObject *kw) +timezone_new_impl(PyTypeObject *type, PyObject *offset, PyObject *name) +/*[clinic end generated code: output=41a2dda500424187 input=d51255afe60382cd]*/ { - PyObject *offset; - PyObject *name = NULL; - if (PyArg_ParseTupleAndKeywords(args, kw, "O!|U:timezone", timezone_kws, - DELTA_TYPE(NO_STATE), &offset, &name)) - return new_timezone(offset, name); - - return NULL; + return new_timezone(offset, name); } static void @@ -4444,9 +4462,6 @@ static PyMethodDef timezone_methods[] = { {NULL, NULL} }; -static const char timezone_doc[] = -PyDoc_STR("Fixed offset from UTC implementation of tzinfo."); - static PyTypeObject PyDateTime_TimeZoneType = { PyVarObject_HEAD_INIT(NULL, 0) "datetime.timezone", /* tp_name */ @@ -4468,7 +4483,7 @@ static PyTypeObject PyDateTime_TimeZoneType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ - timezone_doc, /* tp_doc */ + timezone_new__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ timezone_richcompare, /* tp_richcompare */ @@ -4570,9 +4585,6 @@ static PyGetSetDef time_getset[] = { * Constructors. */ -static char *time_kws[] = {"hour", "minute", "second", "microsecond", - "tzinfo", "fold", NULL}; - static PyObject * time_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) { @@ -4608,17 +4620,10 @@ time_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) static PyObject * time_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *self = NULL; - int hour = 0; - int minute = 0; - int second = 0; - int usecond = 0; - PyObject *tzinfo = Py_None; - int fold = 0; - /* Check for invocation from pickle with __getstate__ state */ if (PyTuple_GET_SIZE(args) >= 1 && PyTuple_GET_SIZE(args) <= 2) { PyObject *state = PyTuple_GET_ITEM(args, 0); + PyObject *tzinfo = Py_None; if (PyTuple_GET_SIZE(args) == 2) { tzinfo = PyTuple_GET_ITEM(args, 1); } @@ -4644,40 +4649,67 @@ time_new(PyTypeObject *type, PyObject *args, PyObject *kw) } return NULL; } - self = time_from_pickle(type, state, tzinfo); + PyObject *self = time_from_pickle(type, state, tzinfo); Py_DECREF(state); return self; } } - tzinfo = Py_None; } - if (PyArg_ParseTupleAndKeywords(args, kw, "|iiiiO$i", time_kws, - &hour, &minute, &second, &usecond, - &tzinfo, &fold)) { - self = new_time_ex2(hour, minute, second, usecond, tzinfo, fold, - type); - } - return self; + return datetime_time(type, args, kw); } -/* Return new time from _strptime.strptime_datetime_time(). */ +/*[clinic input] +@classmethod +datetime.time.__new__ + + hour: int = 0 + minute: int = 0 + second: int = 0 + microsecond: int = 0 + tzinfo: object = None + * + fold: int = 0 + +Time with time zone. + +All arguments are optional. tzinfo may be None, or an instance of +a tzinfo subclass. The remaining arguments may be ints. +[clinic start generated code]*/ + static PyObject * -time_strptime(PyObject *cls, PyObject *args) +datetime_time_impl(PyTypeObject *type, int hour, int minute, int second, + int microsecond, PyObject *tzinfo, int fold) +/*[clinic end generated code: output=f06bb4315225e7f6 input=0148df5e8138fe7b]*/ { - PyObject *string, *format, *result; + return new_time_ex2(hour, minute, second, microsecond, tzinfo, fold, type); +} - if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) { - return NULL; - } +/*[clinic input] +@classmethod +datetime.time.strptime + + string: unicode + format: unicode + / + +Parse string according to the given time format (like time.strptime()). +[clinic start generated code]*/ + +static PyObject * +datetime_time_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format) +/*[clinic end generated code: output=ae05a9bc0241d3bf input=6d0f263a5f94d78d]*/ +{ + PyObject *result; PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { return NULL; } result = PyObject_CallMethodObjArgs(module, - &_Py_ID(_strptime_datetime_time), cls, - string, format, NULL); + &_Py_ID(_strptime_datetime_time), + (PyObject *)type, string, format, NULL); Py_DECREF(module); return result; } @@ -4756,17 +4788,30 @@ time_str(PyObject *op) return PyObject_CallMethodNoArgs(op, &_Py_ID(isoformat)); } +/*[clinic input] +datetime.time.isoformat + + timespec: str(c_default="NULL") = 'auto' + +Return the time formatted according to ISO. + +The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the fractional +part is omitted if self.microsecond == 0. + +The optional argument timespec specifies the number of additional +terms of the time to include. Valid options are 'auto', 'hours', +'minutes', 'seconds', 'milliseconds' and 'microseconds'. +[clinic start generated code]*/ + static PyObject * -time_isoformat(PyObject *op, PyObject *args, PyObject *kw) +datetime_time_isoformat_impl(PyDateTime_Time *self, const char *timespec) +/*[clinic end generated code: output=2bcc7cab65c35545 input=afbbbd953d10ad07]*/ { char buf[100]; - const char *timespec = NULL; - static char *keywords[] = {"timespec", NULL}; - PyDateTime_Time *self = PyTime_CAST(op); PyObject *result; int us = TIME_GET_MICROSECOND(self); - static const char *specs[][2] = { + static const char * const specs[][2] = { {"hours", "%02d"}, {"minutes", "%02d:%02d"}, {"seconds", "%02d:%02d:%02d"}, @@ -4775,9 +4820,6 @@ time_isoformat(PyObject *op, PyObject *args, PyObject *kw) }; size_t given_spec; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|s:isoformat", keywords, ×pec)) - return NULL; - if (timespec == NULL || strcmp(timespec, "auto") == 0) { if (us == 0) { /* seconds */ @@ -4823,18 +4865,22 @@ time_isoformat(PyObject *op, PyObject *args, PyObject *kw) return result; } +/*[clinic input] +datetime.time.strftime + + format: unicode + +Format using strftime(). + +The date part of the timestamp passed to underlying strftime should not be used. +[clinic start generated code]*/ + static PyObject * -time_strftime(PyObject *op, PyObject *args, PyObject *kw) +datetime_time_strftime_impl(PyDateTime_Time *self, PyObject *format) +/*[clinic end generated code: output=10f65af20e2a78c7 input=7dd9df1acbf37b50]*/ { PyObject *result; PyObject *tuple; - PyObject *format; - static char *keywords[] = {"format", NULL}; - PyDateTime_Time *self = PyTime_CAST(op); - - if (! PyArg_ParseTupleAndKeywords(args, kw, "U:strftime", keywords, - &format)) - return NULL; /* Python's strftime does insane things with the year part of the * timetuple. The year is forced to (the otherwise nonsensical) @@ -4855,6 +4901,27 @@ time_strftime(PyObject *op, PyObject *args, PyObject *kw) return result; } +/*[clinic input] +datetime.time.__format__ + + self: self(type="PyObject *") + format: unicode + / + +Formats self with strftime. +[clinic start generated code]*/ + +static PyObject * +datetime_time___format___impl(PyObject *self, PyObject *format) +/*[clinic end generated code: output=4646451f7a5d2156 input=6a858ae787d20230]*/ +{ + /* if the format is zero length, return str(self) */ + if (PyUnicode_GetLength(format) == 0) + return PyObject_Str(self); + + return PyObject_CallMethodOneArg(self, &_Py_ID(strftime), format); +} + /* * Miscellaneous methods. */ @@ -5007,20 +5074,25 @@ datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, /*[clinic end generated code: output=0b89a44c299e4f80 input=abf23656e8df4e97]*/ { return new_time_subclass_fold_ex(hour, minute, second, microsecond, tzinfo, - fold, (PyObject *)Py_TYPE(self)); + fold, Py_TYPE(self)); } -static PyObject * -time_fromisoformat(PyObject *cls, PyObject *tstr) { - assert(tstr != NULL); +/*[clinic input] +@classmethod +datetime.time.fromisoformat - if (!PyUnicode_Check(tstr)) { - PyErr_SetString(PyExc_TypeError, "fromisoformat: argument must be str"); - return NULL; - } + string: unicode + / + +Construct a time from a string in ISO 8601 format. +[clinic start generated code]*/ +static PyObject * +datetime_time_fromisoformat_impl(PyTypeObject *type, PyObject *string) +/*[clinic end generated code: output=97c57e896e7f2535 input=bdb4b8abea9cd688]*/ +{ Py_ssize_t len; - const char *p = PyUnicode_AsUTF8AndSize(tstr, &len); + const char *p = PyUnicode_AsUTF8AndSize(string, &len); if (p == NULL) { goto invalid_string_error; @@ -5063,10 +5135,10 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { } PyObject *t; - if ( (PyTypeObject *)cls == TIME_TYPE(NO_STATE)) { + if (type == TIME_TYPE(NO_STATE)) { t = new_time(hour, minute, second, microsecond, tzinfo, 0); } else { - t = PyObject_CallFunction(cls, "iiiiO", + t = PyObject_CallFunction((PyObject *)type, "iiiiO", hour, minute, second, microsecond, tzinfo); } @@ -5078,7 +5150,7 @@ time_fromisoformat(PyObject *cls, PyObject *tstr) { return NULL; invalid_string_error: - PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", tstr); + PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", string); return NULL; error: @@ -5114,21 +5186,28 @@ time_getstate(PyDateTime_Time *self, int proto) return result; } +/*[clinic input] +datetime.time.__reduce_ex__ + + proto: int + / +[clinic start generated code]*/ + static PyObject * -time_reduce_ex(PyObject *op, PyObject *args) +datetime_time___reduce_ex___impl(PyDateTime_Time *self, int proto) +/*[clinic end generated code: output=ccfab65f5c320c1b input=4cd06bb3ac3657bb]*/ { - int proto; - if (!PyArg_ParseTuple(args, "i:__reduce_ex__", &proto)) - return NULL; - - PyDateTime_Time *self = PyTime_CAST(op); return Py_BuildValue("(ON)", Py_TYPE(self), time_getstate(self, proto)); } +/*[clinic input] +datetime.time.__reduce__ +[clinic start generated code]*/ + static PyObject * -time_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) +datetime_time___reduce___impl(PyDateTime_Time *self) +/*[clinic end generated code: output=9a2fcc87e64ce300 input=0fb8dd14d275857f]*/ { - PyDateTime_Time *self = PyTime_CAST(op); return Py_BuildValue("(ON)", Py_TYPE(self), time_getstate(self, 2)); } @@ -5136,26 +5215,14 @@ static PyMethodDef time_methods[] = { /* Class method: */ - {"strptime", time_strptime, - METH_VARARGS | METH_CLASS, - PyDoc_STR("string, format -> new time parsed from a string " - "(like time.strptime()).")}, + DATETIME_TIME_FROMISOFORMAT_METHODDEF + DATETIME_TIME_STRPTIME_METHODDEF /* Instance methods: */ - {"isoformat", _PyCFunction_CAST(time_isoformat), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("Return string in ISO 8601 format, [HH[:MM[:SS[.mmm[uuu]]]]]" - "[+HH:MM].\n\n" - "The optional argument timespec specifies the number " - "of additional terms\nof the time to include. Valid " - "options are 'auto', 'hours', 'minutes',\n'seconds', " - "'milliseconds' and 'microseconds'.\n")}, - - {"strftime", _PyCFunction_CAST(time_strftime), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("format -> strftime() style string.")}, - - {"__format__", date_format, METH_VARARGS, - PyDoc_STR("Formats self with strftime.")}, + DATETIME_TIME_ISOFORMAT_METHODDEF + DATETIME_TIME_STRFTIME_METHODDEF + DATETIME_TIME___FORMAT___METHODDEF {"utcoffset", time_utcoffset, METH_NOARGS, PyDoc_STR("Return self.tzinfo.utcoffset(self).")}, @@ -5171,24 +5238,12 @@ static PyMethodDef time_methods[] = { {"__replace__", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, - {"fromisoformat", time_fromisoformat, METH_O | METH_CLASS, - PyDoc_STR("string -> time from a string in ISO 8601 format")}, - - {"__reduce_ex__", time_reduce_ex, METH_VARARGS, - PyDoc_STR("__reduce_ex__(proto) -> (cls, state)")}, - - {"__reduce__", time_reduce, METH_NOARGS, - PyDoc_STR("__reduce__() -> (cls, state)")}, + DATETIME_TIME___REDUCE_EX___METHODDEF + DATETIME_TIME___REDUCE___METHODDEF {NULL, NULL} }; -static const char time_doc[] = -PyDoc_STR("time([hour[, minute[, second[, microsecond[, tzinfo]]]]]) --> a time object\n\ -\n\ -All arguments are optional. tzinfo may be None, or an instance of\n\ -a tzinfo subclass. The remaining arguments may be ints.\n"); - static PyTypeObject PyDateTime_TimeType = { PyVarObject_HEAD_INIT(NULL, 0) "datetime.time", /* tp_name */ @@ -5209,8 +5264,8 @@ static PyTypeObject PyDateTime_TimeType = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - time_doc, /* tp_doc */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + datetime_time__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ time_richcompare, /* tp_richcompare */ @@ -5296,11 +5351,6 @@ static PyGetSetDef datetime_getset[] = { * Constructors. */ -static char *datetime_kws[] = { - "year", "month", "day", "hour", "minute", "second", - "microsecond", "tzinfo", "fold", NULL -}; - static PyObject * datetime_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) { @@ -5336,20 +5386,10 @@ datetime_from_pickle(PyTypeObject *type, PyObject *state, PyObject *tzinfo) static PyObject * datetime_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - PyObject *self = NULL; - int year; - int month; - int day; - int hour = 0; - int minute = 0; - int second = 0; - int usecond = 0; - int fold = 0; - PyObject *tzinfo = Py_None; - /* Check for invocation from pickle with __getstate__ state */ if (PyTuple_GET_SIZE(args) >= 1 && PyTuple_GET_SIZE(args) <= 2) { PyObject *state = PyTuple_GET_ITEM(args, 0); + PyObject *tzinfo = Py_None; if (PyTuple_GET_SIZE(args) == 2) { tzinfo = PyTuple_GET_ITEM(args, 1); } @@ -5375,22 +5415,46 @@ datetime_new(PyTypeObject *type, PyObject *args, PyObject *kw) } return NULL; } - self = datetime_from_pickle(type, state, tzinfo); + PyObject *self = datetime_from_pickle(type, state, tzinfo); Py_DECREF(state); return self; } } - tzinfo = Py_None; } - if (PyArg_ParseTupleAndKeywords(args, kw, "iii|iiiiO$i", datetime_kws, - &year, &month, &day, &hour, &minute, - &second, &usecond, &tzinfo, &fold)) { - self = new_datetime_ex2(year, month, day, - hour, minute, second, usecond, - tzinfo, fold, type); - } - return self; + return datetime_datetime(type, args, kw); +} + +/*[clinic input] +@classmethod +datetime.datetime.__new__ + + year: int + month: int + day: int + hour: int = 0 + minute: int = 0 + second: int = 0 + microsecond: int = 0 + tzinfo: object = None + * + fold: int = 0 + +A combination of a date and a time. + +The year, month and day arguments are required. tzinfo may be None, or an +instance of a tzinfo subclass. The remaining arguments may be ints. +[clinic start generated code]*/ + +static PyObject * +datetime_datetime_impl(PyTypeObject *type, int year, int month, int day, + int hour, int minute, int second, int microsecond, + PyObject *tzinfo, int fold) +/*[clinic end generated code: output=47983ddb47d36037 input=2af468d7a9c1e568]*/ +{ + return new_datetime_ex2(year, month, day, + hour, minute, second, microsecond, + tzinfo, fold, type); } /* TM_FUNC is the shared type of _PyTime_localtime() and @@ -5447,7 +5511,7 @@ local(long long u) * Pass localtime or gmtime for f, to control the interpretation of timet. */ static PyObject * -datetime_from_timet_and_us(PyObject *cls, TM_FUNC f, time_t timet, int us, +datetime_from_timet_and_us(PyTypeObject *cls, TM_FUNC f, time_t timet, int us, PyObject *tzinfo) { struct tm tm; @@ -5519,7 +5583,7 @@ datetime_from_timet_and_us(PyObject *cls, TM_FUNC f, time_t timet, int us, * to get that much precision (e.g., C time() isn't good enough). */ static PyObject * -datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, +datetime_from_timestamp(PyTypeObject *cls, TM_FUNC f, PyObject *timestamp, PyObject *tzinfo) { time_t timet; @@ -5537,7 +5601,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, * gmtime for f as appropriate. */ static PyObject * -datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) +datetime_best_possible(PyTypeObject *cls, TM_FUNC f, PyObject *tzinfo) { PyTime_t ts; if (PyTime_Time(&ts) < 0) { @@ -5580,7 +5644,7 @@ datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz) if (check_tzinfo_subclass(tz) < 0) return NULL; - self = datetime_best_possible((PyObject *)type, + self = datetime_best_possible(type, tz == Py_None ? _PyTime_localtime : _PyTime_gmtime, tz); @@ -5596,8 +5660,16 @@ datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz) /* Return best possible UTC time -- this isn't constrained by the * precision of a timestamp. */ +/*[clinic input] +@classmethod +datetime.datetime.utcnow + +Return a new datetime representing UTC day and time. +[clinic start generated code]*/ + static PyObject * -datetime_utcnow(PyObject *cls, PyObject *dummy) +datetime_datetime_utcnow_impl(PyTypeObject *type) +/*[clinic end generated code: output=cfcfe71c6c916ba9 input=576eff2b222b80a1]*/ { if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.datetime.utcnow() is deprecated and scheduled for removal in a " @@ -5606,25 +5678,32 @@ datetime_utcnow(PyObject *cls, PyObject *dummy) { return NULL; } - return datetime_best_possible(cls, _PyTime_gmtime, Py_None); + return datetime_best_possible(type, _PyTime_gmtime, Py_None); } -/* Return new local datetime from timestamp (Python timestamp -- a double). */ +/*[clinic input] +@classmethod +datetime.datetime.fromtimestamp + + timestamp: object + tz as tzinfo: object = None + +Create a datetime from a POSIX timestamp. + +The timestamp is a number, e.g. created via time.time(), that is interpreted +as local time. +[clinic start generated code]*/ + static PyObject * -datetime_fromtimestamp(PyObject *cls, PyObject *args, PyObject *kw) +datetime_datetime_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp, + PyObject *tzinfo) +/*[clinic end generated code: output=9c47ea2b2ebdaded input=34721a5facc94215]*/ { PyObject *self; - PyObject *timestamp; - PyObject *tzinfo = Py_None; - static char *keywords[] = {"timestamp", "tz", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kw, "O|O:fromtimestamp", - keywords, ×tamp, &tzinfo)) - return NULL; if (check_tzinfo_subclass(tzinfo) < 0) return NULL; - self = datetime_from_timestamp(cls, + self = datetime_from_timestamp(type, tzinfo == Py_None ? _PyTime_localtime : _PyTime_gmtime, timestamp, @@ -5638,9 +5717,35 @@ datetime_fromtimestamp(PyObject *cls, PyObject *args, PyObject *kw) return self; } -/* Return new UTC datetime from timestamp (Python timestamp -- a double). */ +/* This is a wrapper for API compatibility with the public C API. */ +static PyObject * +datetime_datetime_fromtimestamp_capi(PyObject *cls, PyObject *args, PyObject *kw) +{ + PyObject *timestamp; + PyObject *tzinfo = Py_None; + static char *keywords[] = {"timestamp", "tz", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kw, "O|O:fromtimestamp", + keywords, ×tamp, &tzinfo)) + return NULL; + return datetime_datetime_fromtimestamp_impl((PyTypeObject *)cls, + timestamp, tzinfo); +} + +/*[clinic input] +@classmethod +datetime.datetime.utcfromtimestamp + + timestamp: object + / + +Create a naive UTC datetime from a POSIX timestamp. +[clinic start generated code]*/ + static PyObject * -datetime_utcfromtimestamp(PyObject *cls, PyObject *args) +datetime_datetime_utcfromtimestamp_impl(PyTypeObject *type, + PyObject *timestamp) +/*[clinic end generated code: output=66d0b1741d788fd2 input=13fabd4296b1c206]*/ { if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.datetime.utcfromtimestamp() is deprecated and scheduled for removal " @@ -5649,23 +5754,27 @@ datetime_utcfromtimestamp(PyObject *cls, PyObject *args) { return NULL; } - PyObject *timestamp; - PyObject *result = NULL; - if (PyArg_ParseTuple(args, "O:utcfromtimestamp", ×tamp)) - result = datetime_from_timestamp(cls, _PyTime_gmtime, timestamp, - Py_None); - return result; + return datetime_from_timestamp(type, _PyTime_gmtime, timestamp, Py_None); } -/* Return new datetime from _strptime.strptime_datetime_datetime(). */ +/*[clinic input] +@classmethod +datetime.datetime.strptime + + string: unicode + format: unicode + / + +Parse string according to the given date and time format (like time.strptime()). +[clinic start generated code]*/ + static PyObject * -datetime_strptime(PyObject *cls, PyObject *args) +datetime_datetime_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format) +/*[clinic end generated code: output=af2c2d024f3203f5 input=b3918835524a1f22]*/ { - PyObject *string, *format, *result; - - if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) - return NULL; + PyObject *result; PyObject *module = PyImport_Import(&_Py_ID(_strptime)); if (module == NULL) { @@ -5673,42 +5782,43 @@ datetime_strptime(PyObject *cls, PyObject *args) } result = PyObject_CallMethodObjArgs(module, &_Py_ID(_strptime_datetime_datetime), - cls, string, format, NULL); + (PyObject *)type, string, format, NULL); Py_DECREF(module); return result; } -/* Return new datetime from date/datetime and time arguments. */ +/*[clinic input] +@classmethod +datetime.datetime.combine + + date: object(subclass_of="DATE_TYPE(NO_STATE)") + time: object(subclass_of="TIME_TYPE(NO_STATE)") + tzinfo: object = NULL + +Construct a datetime from a given date and a given time. +[clinic start generated code]*/ + static PyObject * -datetime_combine(PyObject *cls, PyObject *args, PyObject *kw) +datetime_datetime_combine_impl(PyTypeObject *type, PyObject *date, + PyObject *time, PyObject *tzinfo) +/*[clinic end generated code: output=a10f3cbb90f4d0aa input=4fcf0743288d0bab]*/ { - static char *keywords[] = {"date", "time", "tzinfo", NULL}; - PyObject *date; - PyObject *time; - PyObject *tzinfo = NULL; - PyObject *result = NULL; - - if (PyArg_ParseTupleAndKeywords(args, kw, "O!O!|O:combine", keywords, - DATE_TYPE(NO_STATE), &date, - TIME_TYPE(NO_STATE), &time, &tzinfo)) { - if (tzinfo == NULL) { - if (HASTZINFO(time)) - tzinfo = ((PyDateTime_Time *)time)->tzinfo; - else - tzinfo = Py_None; - } - result = new_datetime_subclass_fold_ex(GET_YEAR(date), - GET_MONTH(date), - GET_DAY(date), - TIME_GET_HOUR(time), - TIME_GET_MINUTE(time), - TIME_GET_SECOND(time), - TIME_GET_MICROSECOND(time), - tzinfo, - TIME_GET_FOLD(time), - cls); + if (tzinfo == NULL) { + if (HASTZINFO(time)) + tzinfo = ((PyDateTime_Time *)time)->tzinfo; + else + tzinfo = Py_None; } - return result; + return new_datetime_subclass_fold_ex(GET_YEAR(date), + GET_MONTH(date), + GET_DAY(date), + TIME_GET_HOUR(time), + TIME_GET_MINUTE(time), + TIME_GET_SECOND(time), + TIME_GET_MICROSECOND(time), + tzinfo, + TIME_GET_FOLD(time), + type); } static PyObject * @@ -5866,23 +5976,26 @@ _find_isoformat_datetime_separator(const char *dtstr, Py_ssize_t len) { } } -static PyObject * -datetime_fromisoformat(PyObject *cls, PyObject *dtstr) -{ - assert(dtstr != NULL); +/*[clinic input] +@classmethod +datetime.datetime.fromisoformat - if (!PyUnicode_Check(dtstr)) { - PyErr_SetString(PyExc_TypeError, - "fromisoformat: argument must be str"); - return NULL; - } + string: unicode + / +Construct a date from a string in ISO 8601 format. +[clinic start generated code]*/ + +static PyObject * +datetime_datetime_fromisoformat_impl(PyTypeObject *type, PyObject *string) +/*[clinic end generated code: output=1800a952fcab79d9 input=d517b158209ded42]*/ +{ // We only need to sanitize this string if the separator is a surrogate // character. In the situation where the separator location is ambiguous, // we don't have to sanitize it anything because that can only happen when // the separator is either '-' or a number. This should mostly be a noop // but it makes the reference counting easier if we still sanitize. - PyObject *dtstr_clean = _sanitize_isoformat_str(dtstr); + PyObject *dtstr_clean = _sanitize_isoformat_str(string); if (dtstr_clean == NULL) { goto invalid_string_error; } @@ -5970,7 +6083,7 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) } } PyObject *dt = new_datetime_subclass_ex(year, month, day, hour, minute, - second, microsecond, tzinfo, cls); + second, microsecond, tzinfo, type); Py_DECREF(tzinfo); Py_DECREF(dtstr_clean); @@ -5983,7 +6096,7 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) return NULL; invalid_string_error: - PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", dtstr); + PyErr_Format(PyExc_ValueError, "Invalid isoformat string: %R", string); error: Py_XDECREF(dtstr_clean); @@ -6060,7 +6173,7 @@ add_datetime_timedelta(PyDateTime_DateTime *date, PyDateTime_Delta *delta, return new_datetime_subclass_ex(year, month, day, hour, minute, second, microsecond, HASTZINFO(date) ? date->tzinfo : Py_None, - (PyObject *)Py_TYPE(date)); + Py_TYPE(date)); } static PyObject * @@ -6222,18 +6335,38 @@ datetime_str(PyObject *op) return res; } +/*[clinic input] +datetime.datetime.isoformat + + sep: int(accept={str}, c_default="'T'", py_default="'T'") = ord('T') + timespec: str(c_default="NULL") = 'auto' + +Return the time formatted according to ISO. + +The full format looks like 'YYYY-MM-DD HH:MM:SS.mmmmmm'. +By default, the fractional part is omitted if self.microsecond == 0. + +If self.tzinfo is not None, the UTC offset is also attached, giving +a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. + +Optional argument sep specifies the separator between date and +time, default 'T'. + +The optional argument timespec specifies the number of additional +terms of the time to include. Valid options are 'auto', 'hours', +'minutes', 'seconds', 'milliseconds' and 'microseconds'. +[clinic start generated code]*/ + static PyObject * -datetime_isoformat(PyObject *op, PyObject *args, PyObject *kw) +datetime_datetime_isoformat_impl(PyDateTime_DateTime *self, int sep, + const char *timespec) +/*[clinic end generated code: output=9b6ce1383189b0bf input=2fa2512172ccf5d5]*/ { - int sep = 'T'; - char *timespec = NULL; - static char *keywords[] = {"sep", "timespec", NULL}; char buffer[100]; - PyDateTime_DateTime *self = PyDateTime_CAST(op); PyObject *result = NULL; int us = DATE_GET_MICROSECOND(self); - static const char *specs[][2] = { + static const char * const specs[][2] = { {"hours", "%04d-%02d-%02d%c%02d"}, {"minutes", "%04d-%02d-%02d%c%02d:%02d"}, {"seconds", "%04d-%02d-%02d%c%02d:%02d:%02d"}, @@ -6242,9 +6375,6 @@ datetime_isoformat(PyObject *op, PyObject *args, PyObject *kw) }; size_t given_spec; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|Cs:isoformat", keywords, &sep, ×pec)) - return NULL; - if (timespec == NULL || strcmp(timespec, "auto") == 0) { if (us == 0) { /* seconds */ @@ -6282,7 +6412,7 @@ datetime_isoformat(PyObject *op, PyObject *args, PyObject *kw) return result; /* We need to append the UTC offset. */ - if (format_utcoffset(buffer, sizeof(buffer), ":", self->tzinfo, op) < 0) { + if (format_utcoffset(buffer, sizeof(buffer), ":", self->tzinfo, (PyObject *)self) < 0) { Py_DECREF(result); return NULL; } @@ -6412,8 +6542,7 @@ datetime_richcompare(PyObject *self, PyObject *other, int op) PyDateTime_Delta *delta; assert(offset1 != offset2); /* else last "if" handled it */ - delta = (PyDateTime_Delta *)datetime_subtract((PyObject *)self, - other); + delta = (PyDateTime_Delta *)datetime_subtract(self, other); if (delta == NULL) goto done; diff = GET_TD_DAYS(delta); @@ -6537,7 +6666,7 @@ datetime_datetime_replace_impl(PyDateTime_DateTime *self, int year, { return new_datetime_subclass_fold_ex(year, month, day, hour, minute, second, microsecond, tzinfo, fold, - (PyObject *)Py_TYPE(self)); + Py_TYPE(self)); } static PyObject * @@ -6673,20 +6802,23 @@ local_timezone_from_local(PyDateTime_DateTime *local_dt) return local_timezone_from_timestamp(timestamp); } +/*[clinic input] +datetime.datetime.astimezone + + tz as tzinfo: object = None + +Convert to local time in new timezone tz. +[clinic start generated code]*/ + static PyObject * -datetime_astimezone(PyObject *op, PyObject *args, PyObject *kw) +datetime_datetime_astimezone_impl(PyDateTime_DateTime *self, + PyObject *tzinfo) +/*[clinic end generated code: output=ae2263d04e944537 input=9c675c8595009935]*/ { - PyDateTime_DateTime *self = PyDateTime_CAST(op); PyDateTime_DateTime *result; PyObject *offset; PyObject *temp; PyObject *self_tzinfo; - PyObject *tzinfo = Py_None; - static char *keywords[] = {"tz", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kw, "|O:astimezone", keywords, - &tzinfo)) - return NULL; if (check_tzinfo_subclass(tzinfo) == -1) return NULL; @@ -6983,22 +7115,29 @@ datetime_getstate(PyDateTime_DateTime *self, int proto) return result; } +/*[clinic input] +datetime.datetime.__reduce_ex__ + + proto: int + / +[clinic start generated code]*/ + static PyObject * -datetime_reduce_ex(PyObject *op, PyObject *args) +datetime_datetime___reduce_ex___impl(PyDateTime_DateTime *self, int proto) +/*[clinic end generated code: output=53d712ce3e927735 input=bab748e49ffb30c3]*/ { - int proto; - if (!PyArg_ParseTuple(args, "i:__reduce_ex__", &proto)) - return NULL; - - PyDateTime_DateTime *self = PyDateTime_CAST(op); return Py_BuildValue("(ON)", Py_TYPE(self), datetime_getstate(self, proto)); } +/*[clinic input] +datetime.datetime.__reduce__ +[clinic start generated code]*/ + static PyObject * -datetime_reduce(PyObject *op, PyObject *Py_UNUSED(arg)) +datetime_datetime___reduce___impl(PyDateTime_DateTime *self) +/*[clinic end generated code: output=6794df9ea75666cf input=cadbbeb3bf3bf94c]*/ { - PyDateTime_DateTime *self = PyDateTime_CAST(op); return Py_BuildValue("(ON)", Py_TYPE(self), datetime_getstate(self, 2)); } @@ -7008,31 +7147,12 @@ static PyMethodDef datetime_methods[] = { /* Class methods: */ DATETIME_DATETIME_NOW_METHODDEF - - {"utcnow", datetime_utcnow, - METH_NOARGS | METH_CLASS, - PyDoc_STR("Return a new datetime representing UTC day and time.")}, - - {"fromtimestamp", _PyCFunction_CAST(datetime_fromtimestamp), - METH_VARARGS | METH_KEYWORDS | METH_CLASS, - PyDoc_STR("timestamp[, tz] -> tz's local time from POSIX timestamp.")}, - - {"utcfromtimestamp", datetime_utcfromtimestamp, - METH_VARARGS | METH_CLASS, - PyDoc_STR("Construct a naive UTC datetime from a POSIX timestamp.")}, - - {"strptime", datetime_strptime, - METH_VARARGS | METH_CLASS, - PyDoc_STR("string, format -> new datetime parsed from a string " - "(like time.strptime()).")}, - - {"combine", _PyCFunction_CAST(datetime_combine), - METH_VARARGS | METH_KEYWORDS | METH_CLASS, - PyDoc_STR("date, time -> datetime with same date and time fields")}, - - {"fromisoformat", datetime_fromisoformat, - METH_O | METH_CLASS, - PyDoc_STR("string -> datetime from a string in most ISO 8601 formats")}, + DATETIME_DATETIME_UTCNOW_METHODDEF + DATETIME_DATETIME_FROMTIMESTAMP_METHODDEF + DATETIME_DATETIME_UTCFROMTIMESTAMP_METHODDEF + DATETIME_DATETIME_STRPTIME_METHODDEF + DATETIME_DATETIME_COMBINE_METHODDEF + DATETIME_DATETIME_FROMISOFORMAT_METHODDEF /* Instance methods: */ @@ -7057,15 +7177,7 @@ static PyMethodDef datetime_methods[] = { {"utctimetuple", datetime_utctimetuple, METH_NOARGS, PyDoc_STR("Return UTC time tuple, compatible with time.localtime().")}, - {"isoformat", _PyCFunction_CAST(datetime_isoformat), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("[sep] -> string in ISO 8601 format, " - "YYYY-MM-DDT[HH[:MM[:SS[.mmm[uuu]]]]][+HH:MM].\n" - "sep is used to separate the year from the time, and " - "defaults to 'T'.\n" - "The optional argument timespec specifies the number " - "of additional terms\nof the time to include. Valid " - "options are 'auto', 'hours', 'minutes',\n'seconds', " - "'milliseconds' and 'microseconds'.\n")}, + DATETIME_DATETIME_ISOFORMAT_METHODDEF {"utcoffset", datetime_utcoffset, METH_NOARGS, PyDoc_STR("Return self.tzinfo.utcoffset(self).")}, @@ -7081,24 +7193,13 @@ static PyMethodDef datetime_methods[] = { {"__replace__", _PyCFunction_CAST(datetime_datetime_replace), METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, - {"astimezone", _PyCFunction_CAST(datetime_astimezone), METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("tz -> convert to local time in new timezone tz\n")}, - - {"__reduce_ex__", datetime_reduce_ex, METH_VARARGS, - PyDoc_STR("__reduce_ex__(proto) -> (cls, state)")}, - - {"__reduce__", datetime_reduce, METH_NOARGS, - PyDoc_STR("__reduce__() -> (cls, state)")}, + DATETIME_DATETIME_ASTIMEZONE_METHODDEF + DATETIME_DATETIME___REDUCE_EX___METHODDEF + DATETIME_DATETIME___REDUCE___METHODDEF {NULL, NULL} }; -static const char datetime_doc[] = -PyDoc_STR("datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])\n\ -\n\ -The year, month and day arguments are required. tzinfo may be None, or an\n\ -instance of a tzinfo subclass. The remaining arguments may be ints.\n"); - static PyNumberMethods datetime_as_number = { datetime_add, /* nb_add */ datetime_subtract, /* nb_subtract */ @@ -7132,8 +7233,8 @@ static PyTypeObject PyDateTime_DateTimeType = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - datetime_doc, /* tp_doc */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + datetime_datetime__doc__, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ datetime_richcompare, /* tp_richcompare */ @@ -7190,7 +7291,7 @@ static PyDateTime_CAPI capi = { .Time_FromTime = new_time_ex, .Delta_FromDelta = new_delta_ex, .TimeZone_FromTimeZone = new_timezone, - .DateTime_FromTimestamp = datetime_fromtimestamp, + .DateTime_FromTimestamp = datetime_datetime_fromtimestamp_capi, .Date_FromTimestamp = datetime_date_fromtimestamp_capi, .DateTime_FromDateAndTimeAndFold = new_datetime_ex2, .Time_FromTimeAndFold = new_time_ex2, diff --git a/Modules/_hashlib/hashlib_buffer.c b/Modules/_hashlib/hashlib_buffer.c deleted file mode 100644 index 032f93ad53ad1b..00000000000000 --- a/Modules/_hashlib/hashlib_buffer.c +++ /dev/null @@ -1,65 +0,0 @@ -#include "hashlib_buffer.h" - -int -_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) -{ - if (data != NULL && string == NULL) { - // called as H(data) or H(data=...) - *res = data; - return 1; - } - else if (data == NULL && string != NULL) { - // called as H(string=...) - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "the 'string' keyword parameter is deprecated since " - "Python 3.15 and slated for removal in Python 3.19; " - "use the 'data' keyword parameter or pass the data " - "to hash as a positional argument instead", 1) < 0) - { - *res = NULL; - return -1; - } - *res = string; - return 1; - } - else if (data == NULL && string == NULL) { - // fast path when no data is given - assert(!PyErr_Occurred()); - *res = NULL; - return 0; - } - else { - // called as H(data=..., string) - *res = NULL; - PyErr_SetString(PyExc_TypeError, - "'data' and 'string' are mutually exclusive " - "and support for 'string' keyword parameter " - "is slated for removal in a future version."); - return -1; - } -} - -int -_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view) -{ - if (PyUnicode_Check(obj)) { - PyErr_SetString(PyExc_TypeError, - "Strings must be encoded before hashing"); - return -1; - } - if (!PyObject_CheckBuffer(obj)) { - PyErr_SetString(PyExc_TypeError, - "object supporting the buffer API required"); - return -1; - } - if (PyObject_GetBuffer(obj, view, PyBUF_SIMPLE) == -1) { - return -1; - } - if (view->ndim > 1) { - PyErr_SetString(PyExc_BufferError, - "Buffer must be single dimension"); - PyBuffer_Release(view); - return -1; - } - return 0; -} diff --git a/Modules/_hashlib/hashlib_buffer.h b/Modules/_hashlib/hashlib_buffer.h deleted file mode 100644 index 809f19884f41b7..00000000000000 --- a/Modules/_hashlib/hashlib_buffer.h +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef _HASHLIB_HASHLIB_BUFFER_H -#define _HASHLIB_HASHLIB_BUFFER_H - -#include "Python.h" - -/* - * Allow to use the 'data' or 'string' keyword in hashlib.new() - * and other hash functions named constructors. - * - * - If 'data' and 'string' are both non-NULL, set an exception and return -1. - * - If 'data' and 'string' are both NULL, set '*res' to NULL and return 0. - * - Otherwise, set '*res' to 'data' or 'string' and return 1. A deprecation - * warning is set when 'string' is specified. - * - * The symbol is exported for '_hashlib' and HACL*-based extension modules. - */ -PyAPI_FUNC(int) -_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string); - -/* - * Obtain a buffer view from a buffer-like object 'obj'. - * - * On success, store the result in 'view' and return 0. - * On error, set an exception and return -1. - * - * The symbol is exported for '_hashlib' and HACL*-based extension modules. - */ -PyAPI_FUNC(int) -_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view); - -/* - * Call _Py_hashlib_get_buffer_view() and check if it succeeded. - * - * On error, set an exception and execute the ERRACTION statements. - */ -#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION) \ - do { \ - if (_Py_hashlib_get_buffer_view(OBJ, VIEW) < 0) { \ - assert(PyErr_Occurred()); \ - ERRACTION; \ - } \ - } while (0) - -/* Specialization of GET_BUFFER_VIEW_OR_ERROR() returning NULL on error. */ -#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW) \ - GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL) - -#endif // !_HASHLIB_HASHLIB_BUFFER_H diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h deleted file mode 100644 index 09add71e0c798c..00000000000000 --- a/Modules/_hashlib/hashlib_fetch.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Utilities used when fetching a message digest from a digest-like identifier. - */ - -#ifndef _HASHLIB_HASHLIB_FETCH_H -#define _HASHLIB_HASHLIB_FETCH_H - -#include "Python.h" - -/* - * Internal error messages used for reporting an unsupported hash algorithm. - * The algorithm can be given by its name, a callable or a PEP-247 module. - * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() - * and _hmacmodule.c::find_hash_info(). - */ -#define _Py_HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" -#define _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" - -#endif // !_HASHLIB_HASHLIB_FETCH_H diff --git a/Modules/_hashlib/hashlib_mutex.h b/Modules/_hashlib/hashlib_mutex.h deleted file mode 100644 index d6924a2ef61e81..00000000000000 --- a/Modules/_hashlib/hashlib_mutex.h +++ /dev/null @@ -1,82 +0,0 @@ -#ifndef _HASHLIB_HASHLIB_MUTEX_H -#define _HASHLIB_HASHLIB_MUTEX_H - -#include "Python.h" -#include "pycore_lock.h" // PyMutex - -/* - * Message length above which the GIL is to be released - * when performing hashing operations. - */ -#define HASHLIB_GIL_MINSIZE 2048 - -/* - * Helper code to synchronize access to the hash object when the GIL is - * released around a CPU consuming hashlib operation. - * - * Code accessing a mutable part of the hash object must be enclosed in - * an HASHLIB_{ACQUIRE,RELEASE}_LOCK block or explicitly acquire and release - * the mutex inside a Py_BEGIN_ALLOW_THREADS -- Py_END_ALLOW_THREADS block if - * they wish to release the GIL for an operation. - */ - -#define HASHLIB_OBJECT_HEAD \ - PyObject_HEAD \ - /* Guard against race conditions during incremental update(). */ \ - PyMutex mutex; - -#define HASHLIB_INIT_MUTEX(OBJ) \ - do { \ - (OBJ)->mutex = (PyMutex){0}; \ - } while (0) - -#define HASHLIB_ACQUIRE_LOCK(OBJ) PyMutex_Lock(&(OBJ)->mutex) -#define HASHLIB_RELEASE_LOCK(OBJ) PyMutex_Unlock(&(OBJ)->mutex) - -// Macros for executing code while conditionally holding the GIL. -// -// These only drop the GIL if the lock acquisition itself is likely to -// block. Thus the non-blocking acquire gating the GIL release for a -// blocking lock acquisition. The intent of these macros is to surround -// the assumed always "fast" operations that you aren't releasing the -// GIL around. - -/* - * Execute a suite of C statements 'STATEMENTS'. - * - * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. - */ -#define HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(SIZE, STATEMENTS) \ - do { \ - if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ - Py_BEGIN_ALLOW_THREADS \ - STATEMENTS; \ - Py_END_ALLOW_THREADS \ - } \ - else { \ - STATEMENTS; \ - } \ - } while (0) - -/* - * Lock 'OBJ' and execute a suite of C statements 'STATEMENTS'. - * - * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. - */ -#define HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(OBJ, SIZE, STATEMENTS) \ - do { \ - if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ - Py_BEGIN_ALLOW_THREADS \ - HASHLIB_ACQUIRE_LOCK(OBJ); \ - STATEMENTS; \ - HASHLIB_RELEASE_LOCK(OBJ); \ - Py_END_ALLOW_THREADS \ - } \ - else { \ - HASHLIB_ACQUIRE_LOCK(OBJ); \ - STATEMENTS; \ - HASHLIB_RELEASE_LOCK(OBJ); \ - } \ - } while (0) - -#endif // !_HASHLIB_HASHLIB_MUTEX_H diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 26412cb62430c9..00f98c090b3952 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -24,17 +24,14 @@ #include "Python.h" #include "pycore_hashtable.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED - -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_fetch.h" -#include "_hashlib/hashlib_mutex.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED +#include "hashlib.h" /* EVP is the preferred interface to hashing in OpenSSL */ #include #include -#include // FIPS_mode() +#include // FIPS_mode() /* We use the object interface to discover what hashes OpenSSL supports. */ #include #include @@ -535,7 +532,7 @@ raise_unsupported_algorithm_error(_hashlibstate *state, PyObject *digestmod) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - _Py_HASHLIB_UNSUPPORTED_ALGORITHM, + HASHLIB_UNSUPPORTED_ALGORITHM, digestmod ); } @@ -545,7 +542,7 @@ raise_unsupported_str_algorithm_error(_hashlibstate *state, const char *name) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM, + HASHLIB_UNSUPPORTED_STR_ALGORITHM, name ); } diff --git a/Modules/_json.c b/Modules/_json.c index 7580b589e2d937..e1d6042cb78ab5 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -51,7 +51,7 @@ typedef struct _PyEncoderObject { char sort_keys; char skipkeys; int allow_nan; - PyCFunction fast_encode; + int (*fast_encode)(PyUnicodeWriter *, PyObject *); } PyEncoderObject; #define PyEncoderObject_CAST(op) ((PyEncoderObject *)(op)) @@ -102,8 +102,10 @@ static PyObject * _encoded_const(PyObject *obj); static void raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end); -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj); +static int +_steal_accumulate(PyUnicodeWriter *writer, PyObject *stolen); +static int +encoder_write_string(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj); static PyObject * encoder_encode_float(PyEncoderObject *s, PyObject *obj); @@ -146,22 +148,11 @@ ascii_escape_unichar(Py_UCS4 c, unsigned char *output, Py_ssize_t chars) return chars; } -static PyObject * -ascii_escape_unicode(PyObject *pystr) +static Py_ssize_t +ascii_escape_size(const void *input, int kind, Py_ssize_t input_chars) { - /* Take a PyUnicode pystr and return a new ASCII-only escaped PyUnicode */ Py_ssize_t i; - Py_ssize_t input_chars; Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; - const void *input; - Py_UCS1 *output; - int kind; - - input_chars = PyUnicode_GET_LENGTH(pystr); - input = PyUnicode_DATA(pystr); - kind = PyUnicode_KIND(pystr); /* Compute the output size */ for (i = 0, output_size = 2; i < input_chars; i++) { @@ -181,11 +172,22 @@ ascii_escape_unicode(PyObject *pystr) } if (output_size > PY_SSIZE_T_MAX - d) { PyErr_SetString(PyExc_OverflowError, "string is too long to escape"); - return NULL; + return -1; } output_size += d; } + return output_size; +} + +static PyObject * +ascii_escape_unicode_and_size(const void *input, int kind, Py_ssize_t input_chars, Py_ssize_t output_size) +{ + Py_ssize_t i; + Py_ssize_t chars; + PyObject *rval; + Py_UCS1 *output; + rval = PyUnicode_New(output_size, 127); if (rval == NULL) { return NULL; @@ -210,23 +212,62 @@ ascii_escape_unicode(PyObject *pystr) } static PyObject * -escape_unicode(PyObject *pystr) +ascii_escape_unicode(PyObject *pystr) +{ + /* Take a PyUnicode pystr and return a new ASCII-only escaped PyUnicode */ + Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); + const void *input = PyUnicode_DATA(pystr); + int kind = PyUnicode_KIND(pystr); + + Py_ssize_t output_size = ascii_escape_size(input, kind, input_chars); + if (output_size < 0) { + return NULL; + } + + return ascii_escape_unicode_and_size(input, kind, input_chars, output_size); +} + +static int +write_escaped_ascii(PyUnicodeWriter *writer, PyObject *pystr) { - /* Take a PyUnicode pystr and return a new escaped PyUnicode */ - Py_ssize_t i; Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; const void *input; int kind; - Py_UCS4 maxchar; - maxchar = PyUnicode_MAX_CHAR_VALUE(pystr); input_chars = PyUnicode_GET_LENGTH(pystr); input = PyUnicode_DATA(pystr); kind = PyUnicode_KIND(pystr); + Py_ssize_t output_size = ascii_escape_size(input, kind, input_chars); + if (output_size < 0) { + return -1; + } + + if (output_size == input_chars + 2) { + /* No need to escape anything */ + if (PyUnicodeWriter_WriteChar(writer, '"') < 0) { + return -1; + } + if (PyUnicodeWriter_WriteStr(writer, pystr) < 0) { + return -1; + } + return PyUnicodeWriter_WriteChar(writer, '"'); + } + + PyObject *rval = ascii_escape_unicode_and_size(input, kind, input_chars, output_size); + if (rval == NULL) { + return -1; + } + + return _steal_accumulate(writer, rval); +} + +static Py_ssize_t +escape_size(const void *input, int kind, Py_ssize_t input_chars) +{ + Py_ssize_t i; + Py_ssize_t output_size; + /* Compute the output size */ for (i = 0, output_size = 2; i < input_chars; i++) { Py_UCS4 c = PyUnicode_READ(kind, input, i); @@ -244,11 +285,21 @@ escape_unicode(PyObject *pystr) } if (output_size > PY_SSIZE_T_MAX - d) { PyErr_SetString(PyExc_OverflowError, "string is too long to escape"); - return NULL; + return -1; } output_size += d; } + return output_size; +} + +static PyObject * +escape_unicode_and_size(const void *input, int kind, Py_UCS4 maxchar, Py_ssize_t input_chars, Py_ssize_t output_size) +{ + Py_ssize_t i; + Py_ssize_t chars; + PyObject *rval; + rval = PyUnicode_New(output_size, maxchar); if (rval == NULL) return NULL; @@ -303,6 +354,55 @@ escape_unicode(PyObject *pystr) return rval; } +static PyObject * +escape_unicode(PyObject *pystr) +{ + /* Take a PyUnicode pystr and return a new escaped PyUnicode */ + Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); + const void *input = PyUnicode_DATA(pystr); + int kind = PyUnicode_KIND(pystr); + Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(pystr); + + Py_ssize_t output_size = escape_size(input, kind, input_chars); + if (output_size < 0) { + return NULL; + } + + return escape_unicode_and_size(input, kind, maxchar, input_chars, output_size); +} + +static int +write_escaped_unicode(PyUnicodeWriter *writer, PyObject *pystr) +{ + Py_ssize_t input_chars = PyUnicode_GET_LENGTH(pystr); + const void *input = PyUnicode_DATA(pystr); + int kind = PyUnicode_KIND(pystr); + Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(pystr); + + Py_ssize_t output_size = escape_size(input, kind, input_chars); + if (output_size < 0) { + return -1; + } + + if (output_size == input_chars + 2) { + /* No need to escape anything */ + if (PyUnicodeWriter_WriteChar(writer, '"') < 0) { + return -1; + } + if (PyUnicodeWriter_WriteStr(writer, pystr) < 0) { + return -1; + } + return PyUnicodeWriter_WriteChar(writer, '"'); + } + + PyObject *rval = escape_unicode_and_size(input, kind, maxchar, input_chars, output_size); + if (rval == NULL) { + return -1; + } + + return _steal_accumulate(writer, rval); +} + static void raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end) { @@ -1256,8 +1356,11 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (PyCFunction_Check(s->encoder)) { PyCFunction f = PyCFunction_GetFunction(s->encoder); - if (f == py_encode_basestring_ascii || f == py_encode_basestring) { - s->fast_encode = f; + if (f == py_encode_basestring_ascii) { + s->fast_encode = write_escaped_ascii; + } + else if (f == py_encode_basestring) { + s->fast_encode = write_escaped_unicode; } } @@ -1438,24 +1541,27 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) return PyFloat_Type.tp_repr(obj); } -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj) +static int +encoder_write_string(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *obj) { /* Return the JSON representation of a string */ PyObject *encoded; if (s->fast_encode) { - return s->fast_encode(NULL, obj); + return s->fast_encode(writer, obj); } encoded = PyObject_CallOneArg(s->encoder, obj); - if (encoded != NULL && !PyUnicode_Check(encoded)) { + if (encoded == NULL) { + return -1; + } + if (!PyUnicode_Check(encoded)) { PyErr_Format(PyExc_TypeError, "encoder() must return a string, not %.80s", Py_TYPE(encoded)->tp_name); Py_DECREF(encoded); - return NULL; + return -1; } - return encoded; + return _steal_accumulate(writer, encoded); } static int @@ -1486,10 +1592,7 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, return PyUnicodeWriter_WriteASCII(writer, "false", 5); } else if (PyUnicode_Check(obj)) { - PyObject *encoded = encoder_encode_string(s, obj); - if (encoded == NULL) - return -1; - return _steal_accumulate(writer, encoded); + return encoder_write_string(s, writer, obj); } else if (PyLong_Check(obj)) { if (PyLong_CheckExact(obj)) { @@ -1578,7 +1681,7 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs PyObject *item_separator) { PyObject *keystr = NULL; - PyObject *encoded; + int rv; if (PyUnicode_Check(key)) { keystr = Py_NewRef(key); @@ -1624,13 +1727,10 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs } } - encoded = encoder_encode_string(s, keystr); + rv = encoder_write_string(s, writer, keystr); Py_DECREF(keystr); - if (encoded == NULL) { - return -1; - } - if (_steal_accumulate(writer, encoded) < 0) { + if (rv < 0) { return -1; } if (PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 41e6d48b1dbd9b..17b5220fd6f9e1 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -87,6 +87,41 @@ copy_grouping(const char* s) return result; } +#if defined(MS_WINDOWS) + +// 16 is the number of elements in the szCodePage field +// of the __crt_locale_strings structure. +#define MAX_CP_LEN 15 + +static int +check_locale_name(const char *locale, const char *end) +{ + size_t len = end ? (size_t)(end - locale) : strlen(locale); + const char *dot = memchr(locale, '.', len); + if (dot && locale + len - dot - 1 > MAX_CP_LEN) { + return -1; + } + return 0; +} + +static int +check_locale_name_all(const char *locale) +{ + const char *start = locale; + while (1) { + const char *end = strchr(start, ';'); + if (check_locale_name(start, end) < 0) { + return -1; + } + if (end == NULL) { + break; + } + start = end + 1; + } + return 0; +} +#endif + /*[clinic input] _locale.setlocale @@ -111,6 +146,18 @@ _locale_setlocale_impl(PyObject *module, int category, const char *locale) "invalid locale category"); return NULL; } + if (locale) { + if ((category == LC_ALL + ? check_locale_name_all(locale) + : check_locale_name(locale, NULL)) < 0) + { + /* Debug assertion failure on Windows. + * _Py_BEGIN_SUPPRESS_IPH/_Py_END_SUPPRESS_IPH do not help. */ + PyErr_SetString(get_locale_state(module)->Error, + "unsupported locale setting"); + return NULL; + } + } #endif if (locale) { diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index 79db85405d6e6b..166d925a542352 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -198,7 +198,7 @@ PyDoc_STRVAR(_zstd_ZstdDict_as_prefix__doc__, "1. Prefix is compatible with long distance matching, while dictionary is not.\n" "2. It only works for the first frame, then the compressor/decompressor will\n" " return to no prefix state.\n" -"3. When decompressing, must use the same prefix as when compressing.\""); +"3. When decompressing, must use the same prefix as when compressing."); #if defined(_zstd_ZstdDict_as_prefix_DOCSTR) # undef _zstd_ZstdDict_as_prefix_DOCSTR #endif @@ -222,4 +222,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) { return _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); } -/*[clinic end generated code: output=4696cbc722e5fdfc input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f41d9e2e2cc2928f input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index b00ee05d2f51bf..026d39f68291da 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -157,10 +157,7 @@ _zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd, zd->dict_len); } else { - /* Impossible code path */ - PyErr_SetString(PyExc_SystemError, - "load_d_dict() impossible code path"); - return -1; + Py_UNREACHABLE(); } /* Check error */ diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 35d6ca8e55a265..76e966f5b1b52a 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -119,10 +119,10 @@ ZstdDict_dealloc(PyObject *ob) } PyDoc_STRVAR(ZstdDict_dictid_doc, -"the Zstandard dictionary, an int between 0 and 2**32.\n\n" -"A non-zero value represents an ordinary Zstandard dictionary, " +"The Zstandard dictionary, an int between 0 and 2**32.\n\n" +"A non-zero value represents an ordinary Zstandard dictionary,\n" "conforming to the standardised format.\n\n" -"The special value '0' means a 'raw content' dictionary," +"A value of zero indicates a 'raw content' dictionary,\n" "without any restrictions on format or content."); static PyObject * @@ -210,12 +210,12 @@ compress(dat, zstd_dict=zd.as_prefix) 1. Prefix is compatible with long distance matching, while dictionary is not. 2. It only works for the first frame, then the compressor/decompressor will return to no prefix state. -3. When decompressing, must use the same prefix as when compressing." +3. When decompressing, must use the same prefix as when compressing. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self) -/*[clinic end generated code: output=6f7130c356595a16 input=d59757b0b5a9551a]*/ +/*[clinic end generated code: output=6f7130c356595a16 input=45b3b6110f36d127]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX); } diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 13c969056be354..163f238a4268d0 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -15,12 +15,10 @@ #endif #include "Python.h" -#include "pycore_moduleobject.h" -#include "pycore_strhex.h" // _Py_strhex() +#include "hashlib.h" +#include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" - -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_mutex.h" +#include "pycore_moduleobject.h" // QUICK CPU AUTODETECTION // diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 18e6129fad8a89..7c4bd5503ed56b 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -8,6 +8,206 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +PyDoc_STRVAR(delta_new__doc__, +"timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0,\n" +" hours=0, weeks=0)\n" +"--\n" +"\n" +"Difference between two datetime values.\n" +"\n" +"All arguments are optional and default to 0.\n" +"Arguments may be integers or floats, and may be positive or negative."); + +static PyObject * +delta_new_impl(PyTypeObject *type, PyObject *days, PyObject *seconds, + PyObject *microseconds, PyObject *milliseconds, + PyObject *minutes, PyObject *hours, PyObject *weeks); + +static PyObject * +delta_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 7 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(days), &_Py_ID(seconds), &_Py_ID(microseconds), &_Py_ID(milliseconds), &_Py_ID(minutes), &_Py_ID(hours), &_Py_ID(weeks), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"days", "seconds", "microseconds", "milliseconds", "minutes", "hours", "weeks", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "timedelta", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[7]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *days = NULL; + PyObject *seconds = NULL; + PyObject *microseconds = NULL; + PyObject *milliseconds = NULL; + PyObject *minutes = NULL; + PyObject *hours = NULL; + PyObject *weeks = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 7, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + days = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[1]) { + seconds = fastargs[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[2]) { + microseconds = fastargs[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[3]) { + milliseconds = fastargs[3]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[4]) { + minutes = fastargs[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[5]) { + hours = fastargs[5]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + weeks = fastargs[6]; +skip_optional_pos: + return_value = delta_new_impl(type, days, seconds, microseconds, milliseconds, minutes, hours, weeks); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date__doc__, +"date(year, month, day)\n" +"--\n" +"\n" +"Concrete date type."); + +static PyObject * +datetime_date_impl(PyTypeObject *type, int year, int month, int day); + +static PyObject * +datetime_date(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "date", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + int year; + int month; + int day; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + year = PyLong_AsInt(fastargs[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + month = PyLong_AsInt(fastargs[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + day = PyLong_AsInt(fastargs[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_date_impl(type, year, month, day); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_today__doc__, +"today($type, /)\n" +"--\n" +"\n" +"Current date or datetime.\n" +"\n" +"Equivalent to fromtimestamp(time.time())."); + +#define DATETIME_DATE_TODAY_METHODDEF \ + {"today", (PyCFunction)datetime_date_today, METH_NOARGS|METH_CLASS, datetime_date_today__doc__}, + +static PyObject * +datetime_date_today_impl(PyTypeObject *type); + +static PyObject * +datetime_date_today(PyObject *type, PyObject *Py_UNUSED(ignored)) +{ + return datetime_date_today_impl((PyTypeObject *)type); +} + PyDoc_STRVAR(datetime_date_fromtimestamp__doc__, "fromtimestamp($type, timestamp, /)\n" "--\n" @@ -33,12 +233,83 @@ datetime_date_fromtimestamp(PyObject *type, PyObject *timestamp) return return_value; } +PyDoc_STRVAR(datetime_date_fromordinal__doc__, +"fromordinal($type, ordinal, /)\n" +"--\n" +"\n" +"Construct a date from a proleptic Gregorian ordinal.\n" +"\n" +"January 1 of year 1 is day 1. Only the year, month and day are\n" +"non-zero in the result."); + +#define DATETIME_DATE_FROMORDINAL_METHODDEF \ + {"fromordinal", (PyCFunction)datetime_date_fromordinal, METH_O|METH_CLASS, datetime_date_fromordinal__doc__}, + static PyObject * -iso_calendar_date_new_impl(PyTypeObject *type, int year, int week, - int weekday); +datetime_date_fromordinal_impl(PyTypeObject *type, int ordinal); static PyObject * -iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +datetime_date_fromordinal(PyObject *type, PyObject *arg) +{ + PyObject *return_value = NULL; + int ordinal; + + ordinal = PyLong_AsInt(arg); + if (ordinal == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_date_fromordinal_impl((PyTypeObject *)type, ordinal); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_fromisoformat__doc__, +"fromisoformat($type, string, /)\n" +"--\n" +"\n" +"Construct a date from a string in ISO 8601 format."); + +#define DATETIME_DATE_FROMISOFORMAT_METHODDEF \ + {"fromisoformat", (PyCFunction)datetime_date_fromisoformat, METH_O|METH_CLASS, datetime_date_fromisoformat__doc__}, + +static PyObject * +datetime_date_fromisoformat_impl(PyTypeObject *type, PyObject *string); + +static PyObject * +datetime_date_fromisoformat(PyObject *type, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *string; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("fromisoformat", "argument", "str", arg); + goto exit; + } + string = arg; + return_value = datetime_date_fromisoformat_impl((PyTypeObject *)type, string); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_fromisocalendar__doc__, +"fromisocalendar($type, /, year, week, day)\n" +"--\n" +"\n" +"Construct a date from the ISO year, week number and weekday.\n" +"\n" +"This is the inverse of the date.isocalendar() function."); + +#define DATETIME_DATE_FROMISOCALENDAR_METHODDEF \ + {"fromisocalendar", _PyCFunction_CAST(datetime_date_fromisocalendar), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_date_fromisocalendar__doc__}, + +static PyObject * +datetime_date_fromisocalendar_impl(PyTypeObject *type, int year, int week, + int day); + +static PyObject * +datetime_date_fromisocalendar(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -52,7 +323,7 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(year), &_Py_ID(week), &_Py_ID(weekday), }, + .ob_item = { &_Py_ID(year), &_Py_ID(week), &_Py_ID(day), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -61,58 +332,1293 @@ iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"year", "week", "weekday", NULL}; + static const char * const _keywords[] = {"year", "week", "day", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "IsoCalendarDate", + .fname = "fromisocalendar", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[3]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); int year; int week; - int weekday; + int day; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!fastargs) { + if (!args) { goto exit; } - year = PyLong_AsInt(fastargs[0]); + year = PyLong_AsInt(args[0]); if (year == -1 && PyErr_Occurred()) { goto exit; } - week = PyLong_AsInt(fastargs[1]); + week = PyLong_AsInt(args[1]); if (week == -1 && PyErr_Occurred()) { goto exit; } - weekday = PyLong_AsInt(fastargs[2]); - if (weekday == -1 && PyErr_Occurred()) { + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { goto exit; } - return_value = iso_calendar_date_new_impl(type, year, week, weekday); + return_value = datetime_date_fromisocalendar_impl((PyTypeObject *)type, year, week, day); exit: return return_value; } -PyDoc_STRVAR(datetime_date_replace__doc__, -"replace($self, /, year=unchanged, month=unchanged, day=unchanged)\n" +PyDoc_STRVAR(datetime_date_strptime__doc__, +"strptime($type, string, format, /)\n" "--\n" "\n" -"Return date with new specified fields."); +"Parse string according to the given date format (like time.strptime())."); + +#define DATETIME_DATE_STRPTIME_METHODDEF \ + {"strptime", _PyCFunction_CAST(datetime_date_strptime), METH_FASTCALL|METH_CLASS, datetime_date_strptime__doc__}, + +static PyObject * +datetime_date_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format); + +static PyObject * +datetime_date_strptime(PyObject *type, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *string; + PyObject *format; + + if (!_PyArg_CheckPositional("strptime", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strptime", "argument 1", "str", args[0]); + goto exit; + } + string = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("strptime", "argument 2", "str", args[1]); + goto exit; + } + format = args[1]; + return_value = datetime_date_strptime_impl((PyTypeObject *)type, string, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_strftime__doc__, +"strftime($self, /, format)\n" +"--\n" +"\n" +"Format using strftime().\n" +"\n" +"Example: \"%d/%m/%Y, %H:%M:%S\"."); + +#define DATETIME_DATE_STRFTIME_METHODDEF \ + {"strftime", _PyCFunction_CAST(datetime_date_strftime), METH_FASTCALL|METH_KEYWORDS, datetime_date_strftime__doc__}, + +static PyObject * +datetime_date_strftime_impl(PyObject *self, PyObject *format); + +static PyObject * +datetime_date_strftime(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(format), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"format", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "strftime", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *format; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strftime", "argument 'format'", "str", args[0]); + goto exit; + } + format = args[0]; + return_value = datetime_date_strftime_impl(self, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date___format____doc__, +"__format__($self, format, /)\n" +"--\n" +"\n" +"Formats self with strftime."); + +#define DATETIME_DATE___FORMAT___METHODDEF \ + {"__format__", (PyCFunction)datetime_date___format__, METH_O, datetime_date___format____doc__}, + +static PyObject * +datetime_date___format___impl(PyObject *self, PyObject *format); + +static PyObject * +datetime_date___format__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *format; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("__format__", "argument", "str", arg); + goto exit; + } + format = arg; + return_value = datetime_date___format___impl(self, format); + +exit: + return return_value; +} + +static PyObject * +iso_calendar_date_new_impl(PyTypeObject *type, int year, int week, + int weekday); + +static PyObject * +iso_calendar_date_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(week), &_Py_ID(weekday), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "week", "weekday", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "IsoCalendarDate", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + int year; + int week; + int weekday; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + year = PyLong_AsInt(fastargs[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + week = PyLong_AsInt(fastargs[1]); + if (week == -1 && PyErr_Occurred()) { + goto exit; + } + weekday = PyLong_AsInt(fastargs[2]); + if (weekday == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = iso_calendar_date_new_impl(type, year, week, weekday); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_date_replace__doc__, +"replace($self, /, year=unchanged, month=unchanged, day=unchanged)\n" +"--\n" +"\n" +"Return date with new specified fields."); + +#define DATETIME_DATE_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL|METH_KEYWORDS, datetime_date_replace__doc__}, + +static PyObject * +datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, + int day); + +static PyObject * +datetime_date_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int year = GET_YEAR(self); + int month = GET_MONTH(self); + int day = GET_DAY(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + year = PyLong_AsInt(args[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + month = PyLong_AsInt(args[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + day = PyLong_AsInt(args[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = datetime_date_replace_impl((PyDateTime_Date *)self, year, month, day); + +exit: + return return_value; +} + +PyDoc_STRVAR(timezone_new__doc__, +"timezone(offset, name=)\n" +"--\n" +"\n" +"Fixed offset from UTC implementation of tzinfo."); + +static PyObject * +timezone_new_impl(PyTypeObject *type, PyObject *offset, PyObject *name); + +static PyObject * +timezone_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(offset), &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"offset", "name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "timezone", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; + PyObject *offset; + PyObject *name = NULL; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!PyObject_TypeCheck(fastargs[0], DELTA_TYPE(NO_STATE))) { + _PyArg_BadArgument("timezone", "argument 'offset'", (DELTA_TYPE(NO_STATE))->tp_name, fastargs[0]); + goto exit; + } + offset = fastargs[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (!PyUnicode_Check(fastargs[1])) { + _PyArg_BadArgument("timezone", "argument 'name'", "str", fastargs[1]); + goto exit; + } + name = fastargs[1]; +skip_optional_pos: + return_value = timezone_new_impl(type, offset, name); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time__doc__, +"time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0)\n" +"--\n" +"\n" +"Time with time zone.\n" +"\n" +"All arguments are optional. tzinfo may be None, or an instance of\n" +"a tzinfo subclass. The remaining arguments may be ints."); + +static PyObject * +datetime_time_impl(PyTypeObject *type, int hour, int minute, int second, + int microsecond, PyObject *tzinfo, int fold); + +static PyObject * +datetime_time(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "time", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; + PyObject *tzinfo = Py_None; + int fold = 0; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 0, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + hour = PyLong_AsInt(fastargs[0]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[1]) { + minute = PyLong_AsInt(fastargs[1]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[2]) { + second = PyLong_AsInt(fastargs[2]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[3]) { + microsecond = PyLong_AsInt(fastargs[3]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[4]) { + tzinfo = fastargs[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(fastargs[5]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_time_impl(type, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_strptime__doc__, +"strptime($type, string, format, /)\n" +"--\n" +"\n" +"Parse string according to the given time format (like time.strptime())."); + +#define DATETIME_TIME_STRPTIME_METHODDEF \ + {"strptime", _PyCFunction_CAST(datetime_time_strptime), METH_FASTCALL|METH_CLASS, datetime_time_strptime__doc__}, + +static PyObject * +datetime_time_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format); + +static PyObject * +datetime_time_strptime(PyObject *type, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *string; + PyObject *format; + + if (!_PyArg_CheckPositional("strptime", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strptime", "argument 1", "str", args[0]); + goto exit; + } + string = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("strptime", "argument 2", "str", args[1]); + goto exit; + } + format = args[1]; + return_value = datetime_time_strptime_impl((PyTypeObject *)type, string, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_isoformat__doc__, +"isoformat($self, /, timespec=\'auto\')\n" +"--\n" +"\n" +"Return the time formatted according to ISO.\n" +"\n" +"The full format is \'HH:MM:SS.mmmmmm+zz:zz\'. By default, the fractional\n" +"part is omitted if self.microsecond == 0.\n" +"\n" +"The optional argument timespec specifies the number of additional\n" +"terms of the time to include. Valid options are \'auto\', \'hours\',\n" +"\'minutes\', \'seconds\', \'milliseconds\' and \'microseconds\'."); + +#define DATETIME_TIME_ISOFORMAT_METHODDEF \ + {"isoformat", _PyCFunction_CAST(datetime_time_isoformat), METH_FASTCALL|METH_KEYWORDS, datetime_time_isoformat__doc__}, + +static PyObject * +datetime_time_isoformat_impl(PyDateTime_Time *self, const char *timespec); + +static PyObject * +datetime_time_isoformat(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(timespec), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"timespec", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "isoformat", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + const char *timespec = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("isoformat", "argument 'timespec'", "str", args[0]); + goto exit; + } + Py_ssize_t timespec_length; + timespec = PyUnicode_AsUTF8AndSize(args[0], ×pec_length); + if (timespec == NULL) { + goto exit; + } + if (strlen(timespec) != (size_t)timespec_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } +skip_optional_pos: + return_value = datetime_time_isoformat_impl((PyDateTime_Time *)self, timespec); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_strftime__doc__, +"strftime($self, /, format)\n" +"--\n" +"\n" +"Format using strftime().\n" +"\n" +"The date part of the timestamp passed to underlying strftime should not be used."); + +#define DATETIME_TIME_STRFTIME_METHODDEF \ + {"strftime", _PyCFunction_CAST(datetime_time_strftime), METH_FASTCALL|METH_KEYWORDS, datetime_time_strftime__doc__}, + +static PyObject * +datetime_time_strftime_impl(PyDateTime_Time *self, PyObject *format); + +static PyObject * +datetime_time_strftime(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(format), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"format", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "strftime", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *format; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strftime", "argument 'format'", "str", args[0]); + goto exit; + } + format = args[0]; + return_value = datetime_time_strftime_impl((PyDateTime_Time *)self, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time___format____doc__, +"__format__($self, format, /)\n" +"--\n" +"\n" +"Formats self with strftime."); + +#define DATETIME_TIME___FORMAT___METHODDEF \ + {"__format__", (PyCFunction)datetime_time___format__, METH_O, datetime_time___format____doc__}, + +static PyObject * +datetime_time___format___impl(PyObject *self, PyObject *format); + +static PyObject * +datetime_time___format__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *format; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("__format__", "argument", "str", arg); + goto exit; + } + format = arg; + return_value = datetime_time___format___impl(self, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_replace__doc__, +"replace($self, /, hour=unchanged, minute=unchanged, second=unchanged,\n" +" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +"--\n" +"\n" +"Return time with new specified fields."); + +#define DATETIME_TIME_REPLACE_METHODDEF \ + {"replace", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL|METH_KEYWORDS, datetime_time_replace__doc__}, + +static PyObject * +datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, + int second, int microsecond, PyObject *tzinfo, + int fold); + +static PyObject * +datetime_time_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "replace", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + int hour = TIME_GET_HOUR(self); + int minute = TIME_GET_MINUTE(self); + int second = TIME_GET_SECOND(self); + int microsecond = TIME_GET_MICROSECOND(self); + PyObject *tzinfo = HASTZINFO(self) ? ((PyDateTime_Time *)self)->tzinfo : Py_None; + int fold = TIME_GET_FOLD(self); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + hour = PyLong_AsInt(args[0]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + minute = PyLong_AsInt(args[1]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[2]) { + second = PyLong_AsInt(args[2]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[3]) { + microsecond = PyLong_AsInt(args[3]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[4]) { + tzinfo = args[4]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(args[5]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_time_replace_impl((PyDateTime_Time *)self, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time_fromisoformat__doc__, +"fromisoformat($type, string, /)\n" +"--\n" +"\n" +"Construct a time from a string in ISO 8601 format."); + +#define DATETIME_TIME_FROMISOFORMAT_METHODDEF \ + {"fromisoformat", (PyCFunction)datetime_time_fromisoformat, METH_O|METH_CLASS, datetime_time_fromisoformat__doc__}, + +static PyObject * +datetime_time_fromisoformat_impl(PyTypeObject *type, PyObject *string); + +static PyObject * +datetime_time_fromisoformat(PyObject *type, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *string; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("fromisoformat", "argument", "str", arg); + goto exit; + } + string = arg; + return_value = datetime_time_fromisoformat_impl((PyTypeObject *)type, string); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time___reduce_ex____doc__, +"__reduce_ex__($self, proto, /)\n" +"--\n" +"\n"); + +#define DATETIME_TIME___REDUCE_EX___METHODDEF \ + {"__reduce_ex__", (PyCFunction)datetime_time___reduce_ex__, METH_O, datetime_time___reduce_ex____doc__}, + +static PyObject * +datetime_time___reduce_ex___impl(PyDateTime_Time *self, int proto); + +static PyObject * +datetime_time___reduce_ex__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int proto; + + proto = PyLong_AsInt(arg); + if (proto == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_time___reduce_ex___impl((PyDateTime_Time *)self, proto); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_time___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define DATETIME_TIME___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)datetime_time___reduce__, METH_NOARGS, datetime_time___reduce____doc__}, + +static PyObject * +datetime_time___reduce___impl(PyDateTime_Time *self); + +static PyObject * +datetime_time___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return datetime_time___reduce___impl((PyDateTime_Time *)self); +} + +PyDoc_STRVAR(datetime_datetime__doc__, +"datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0,\n" +" tzinfo=None, *, fold=0)\n" +"--\n" +"\n" +"A combination of a date and a time.\n" +"\n" +"The year, month and day arguments are required. tzinfo may be None, or an\n" +"instance of a tzinfo subclass. The remaining arguments may be ints."); + +static PyObject * +datetime_datetime_impl(PyTypeObject *type, int year, int month, int day, + int hour, int minute, int second, int microsecond, + PyObject *tzinfo, int fold); + +static PyObject * +datetime_datetime(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"year", "month", "day", "hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "datetime", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 3; + int year; + int month; + int day; + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; + PyObject *tzinfo = Py_None; + int fold = 0; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 3, /*maxpos*/ 8, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + year = PyLong_AsInt(fastargs[0]); + if (year == -1 && PyErr_Occurred()) { + goto exit; + } + month = PyLong_AsInt(fastargs[1]); + if (month == -1 && PyErr_Occurred()) { + goto exit; + } + day = PyLong_AsInt(fastargs[2]); + if (day == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[3]) { + hour = PyLong_AsInt(fastargs[3]); + if (hour == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[4]) { + minute = PyLong_AsInt(fastargs[4]); + if (minute == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[5]) { + second = PyLong_AsInt(fastargs[5]); + if (second == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[6]) { + microsecond = PyLong_AsInt(fastargs[6]); + if (microsecond == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (fastargs[7]) { + tzinfo = fastargs[7]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + fold = PyLong_AsInt(fastargs[8]); + if (fold == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: + return_value = datetime_datetime_impl(type, year, month, day, hour, minute, second, microsecond, tzinfo, fold); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_now__doc__, +"now($type, /, tz=None)\n" +"--\n" +"\n" +"Returns new datetime object representing current time local to tz.\n" +"\n" +" tz\n" +" Timezone object.\n" +"\n" +"If no tz is specified, uses local timezone."); + +#define DATETIME_DATETIME_NOW_METHODDEF \ + {"now", _PyCFunction_CAST(datetime_datetime_now), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_now__doc__}, + +static PyObject * +datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz); + +static PyObject * +datetime_datetime_now(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(tz), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"tz", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "now", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *tz = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + tz = args[0]; +skip_optional_pos: + return_value = datetime_datetime_now_impl((PyTypeObject *)type, tz); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_utcnow__doc__, +"utcnow($type, /)\n" +"--\n" +"\n" +"Return a new datetime representing UTC day and time."); + +#define DATETIME_DATETIME_UTCNOW_METHODDEF \ + {"utcnow", (PyCFunction)datetime_datetime_utcnow, METH_NOARGS|METH_CLASS, datetime_datetime_utcnow__doc__}, + +static PyObject * +datetime_datetime_utcnow_impl(PyTypeObject *type); + +static PyObject * +datetime_datetime_utcnow(PyObject *type, PyObject *Py_UNUSED(ignored)) +{ + return datetime_datetime_utcnow_impl((PyTypeObject *)type); +} + +PyDoc_STRVAR(datetime_datetime_fromtimestamp__doc__, +"fromtimestamp($type, /, timestamp, tz=None)\n" +"--\n" +"\n" +"Create a datetime from a POSIX timestamp.\n" +"\n" +"The timestamp is a number, e.g. created via time.time(), that is interpreted\n" +"as local time."); + +#define DATETIME_DATETIME_FROMTIMESTAMP_METHODDEF \ + {"fromtimestamp", _PyCFunction_CAST(datetime_datetime_fromtimestamp), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_fromtimestamp__doc__}, + +static PyObject * +datetime_datetime_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp, + PyObject *tzinfo); + +static PyObject * +datetime_datetime_fromtimestamp(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(timestamp), &_Py_ID(tz), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"timestamp", "tz", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "fromtimestamp", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *timestamp; + PyObject *tzinfo = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + timestamp = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + tzinfo = args[1]; +skip_optional_pos: + return_value = datetime_datetime_fromtimestamp_impl((PyTypeObject *)type, timestamp, tzinfo); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_utcfromtimestamp__doc__, +"utcfromtimestamp($type, timestamp, /)\n" +"--\n" +"\n" +"Create a naive UTC datetime from a POSIX timestamp."); + +#define DATETIME_DATETIME_UTCFROMTIMESTAMP_METHODDEF \ + {"utcfromtimestamp", (PyCFunction)datetime_datetime_utcfromtimestamp, METH_O|METH_CLASS, datetime_datetime_utcfromtimestamp__doc__}, + +static PyObject * +datetime_datetime_utcfromtimestamp_impl(PyTypeObject *type, + PyObject *timestamp); + +static PyObject * +datetime_datetime_utcfromtimestamp(PyObject *type, PyObject *timestamp) +{ + PyObject *return_value = NULL; + + return_value = datetime_datetime_utcfromtimestamp_impl((PyTypeObject *)type, timestamp); + + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_strptime__doc__, +"strptime($type, string, format, /)\n" +"--\n" +"\n" +"Parse string according to the given date and time format (like time.strptime())."); + +#define DATETIME_DATETIME_STRPTIME_METHODDEF \ + {"strptime", _PyCFunction_CAST(datetime_datetime_strptime), METH_FASTCALL|METH_CLASS, datetime_datetime_strptime__doc__}, + +static PyObject * +datetime_datetime_strptime_impl(PyTypeObject *type, PyObject *string, + PyObject *format); + +static PyObject * +datetime_datetime_strptime(PyObject *type, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *string; + PyObject *format; + + if (!_PyArg_CheckPositional("strptime", nargs, 2, 2)) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("strptime", "argument 1", "str", args[0]); + goto exit; + } + string = args[0]; + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("strptime", "argument 2", "str", args[1]); + goto exit; + } + format = args[1]; + return_value = datetime_datetime_strptime_impl((PyTypeObject *)type, string, format); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime_combine__doc__, +"combine($type, /, date, time, tzinfo=)\n" +"--\n" +"\n" +"Construct a datetime from a given date and a given time."); -#define DATETIME_DATE_REPLACE_METHODDEF \ - {"replace", _PyCFunction_CAST(datetime_date_replace), METH_FASTCALL|METH_KEYWORDS, datetime_date_replace__doc__}, +#define DATETIME_DATETIME_COMBINE_METHODDEF \ + {"combine", _PyCFunction_CAST(datetime_datetime_combine), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_combine__doc__}, static PyObject * -datetime_date_replace_impl(PyDateTime_Date *self, int year, int month, - int day); +datetime_datetime_combine_impl(PyTypeObject *type, PyObject *date, + PyObject *time, PyObject *tzinfo); static PyObject * -datetime_date_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +datetime_datetime_combine(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -126,7 +1632,7 @@ datetime_date_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(year), &_Py_ID(month), &_Py_ID(day), }, + .ob_item = { &_Py_ID(date), &_Py_ID(time), &_Py_ID(tzinfo), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -135,200 +1641,107 @@ datetime_date_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"year", "month", "day", NULL}; + static const char * const _keywords[] = {"date", "time", "tzinfo", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "replace", + .fname = "combine", .kwtuple = KWTUPLE, }; #undef KWTUPLE PyObject *argsbuf[3]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - int year = GET_YEAR(self); - int month = GET_MONTH(self); - int day = GET_DAY(self); + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *date; + PyObject *time; + PyObject *tzinfo = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 0, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 2, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - year = PyLong_AsInt(args[0]); - if (year == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (args[1]) { - month = PyLong_AsInt(args[1]); - if (month == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } + if (!PyObject_TypeCheck(args[0], DATE_TYPE(NO_STATE))) { + _PyArg_BadArgument("combine", "argument 'date'", (DATE_TYPE(NO_STATE))->tp_name, args[0]); + goto exit; } - day = PyLong_AsInt(args[2]); - if (day == -1 && PyErr_Occurred()) { + date = args[0]; + if (!PyObject_TypeCheck(args[1], TIME_TYPE(NO_STATE))) { + _PyArg_BadArgument("combine", "argument 'time'", (TIME_TYPE(NO_STATE))->tp_name, args[1]); goto exit; } + time = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + tzinfo = args[2]; skip_optional_pos: - return_value = datetime_date_replace_impl((PyDateTime_Date *)self, year, month, day); + return_value = datetime_datetime_combine_impl((PyTypeObject *)type, date, time, tzinfo); exit: return return_value; } -PyDoc_STRVAR(datetime_time_replace__doc__, -"replace($self, /, hour=unchanged, minute=unchanged, second=unchanged,\n" -" microsecond=unchanged, tzinfo=unchanged, *, fold=unchanged)\n" +PyDoc_STRVAR(datetime_datetime_fromisoformat__doc__, +"fromisoformat($type, string, /)\n" "--\n" "\n" -"Return time with new specified fields."); +"Construct a date from a string in ISO 8601 format."); -#define DATETIME_TIME_REPLACE_METHODDEF \ - {"replace", _PyCFunction_CAST(datetime_time_replace), METH_FASTCALL|METH_KEYWORDS, datetime_time_replace__doc__}, +#define DATETIME_DATETIME_FROMISOFORMAT_METHODDEF \ + {"fromisoformat", (PyCFunction)datetime_datetime_fromisoformat, METH_O|METH_CLASS, datetime_datetime_fromisoformat__doc__}, static PyObject * -datetime_time_replace_impl(PyDateTime_Time *self, int hour, int minute, - int second, int microsecond, PyObject *tzinfo, - int fold); +datetime_datetime_fromisoformat_impl(PyTypeObject *type, PyObject *string); static PyObject * -datetime_time_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +datetime_datetime_fromisoformat(PyObject *type, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 6 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - Py_hash_t ob_hash; - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_hash = -1, - .ob_item = { &_Py_ID(hour), &_Py_ID(minute), &_Py_ID(second), &_Py_ID(microsecond), &_Py_ID(tzinfo), &_Py_ID(fold), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"hour", "minute", "second", "microsecond", "tzinfo", "fold", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "replace", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[6]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - int hour = TIME_GET_HOUR(self); - int minute = TIME_GET_MINUTE(self); - int second = TIME_GET_SECOND(self); - int microsecond = TIME_GET_MICROSECOND(self); - PyObject *tzinfo = HASTZINFO(self) ? ((PyDateTime_Time *)self)->tzinfo : Py_None; - int fold = TIME_GET_FOLD(self); + PyObject *string; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 0, /*maxpos*/ 5, /*minkw*/ 0, /*varpos*/ 0, argsbuf); - if (!args) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - hour = PyLong_AsInt(args[0]); - if (hour == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (args[1]) { - minute = PyLong_AsInt(args[1]); - if (minute == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (args[2]) { - second = PyLong_AsInt(args[2]); - if (second == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (args[3]) { - microsecond = PyLong_AsInt(args[3]); - if (microsecond == -1 && PyErr_Occurred()) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - if (args[4]) { - tzinfo = args[4]; - if (!--noptargs) { - goto skip_optional_pos; - } - } -skip_optional_pos: - if (!noptargs) { - goto skip_optional_kwonly; - } - fold = PyLong_AsInt(args[5]); - if (fold == -1 && PyErr_Occurred()) { + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("fromisoformat", "argument", "str", arg); goto exit; } -skip_optional_kwonly: - return_value = datetime_time_replace_impl((PyDateTime_Time *)self, hour, minute, second, microsecond, tzinfo, fold); + string = arg; + return_value = datetime_datetime_fromisoformat_impl((PyTypeObject *)type, string); exit: return return_value; } -PyDoc_STRVAR(datetime_datetime_now__doc__, -"now($type, /, tz=None)\n" +PyDoc_STRVAR(datetime_datetime_isoformat__doc__, +"isoformat($self, /, sep=\'T\', timespec=\'auto\')\n" "--\n" "\n" -"Returns new datetime object representing current time local to tz.\n" +"Return the time formatted according to ISO.\n" "\n" -" tz\n" -" Timezone object.\n" +"The full format looks like \'YYYY-MM-DD HH:MM:SS.mmmmmm\'.\n" +"By default, the fractional part is omitted if self.microsecond == 0.\n" "\n" -"If no tz is specified, uses local timezone."); +"If self.tzinfo is not None, the UTC offset is also attached, giving\n" +"a full format of \'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM\'.\n" +"\n" +"Optional argument sep specifies the separator between date and\n" +"time, default \'T\'.\n" +"\n" +"The optional argument timespec specifies the number of additional\n" +"terms of the time to include. Valid options are \'auto\', \'hours\',\n" +"\'minutes\', \'seconds\', \'milliseconds\' and \'microseconds\'."); -#define DATETIME_DATETIME_NOW_METHODDEF \ - {"now", _PyCFunction_CAST(datetime_datetime_now), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_now__doc__}, +#define DATETIME_DATETIME_ISOFORMAT_METHODDEF \ + {"isoformat", _PyCFunction_CAST(datetime_datetime_isoformat), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_isoformat__doc__}, static PyObject * -datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz); +datetime_datetime_isoformat_impl(PyDateTime_DateTime *self, int sep, + const char *timespec); static PyObject * -datetime_datetime_now(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +datetime_datetime_isoformat(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -337,7 +1750,7 @@ datetime_datetime_now(PyObject *type, PyObject *const *args, Py_ssize_t nargs, P } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(tz), }, + .ob_item = { &_Py_ID(sep), &_Py_ID(timespec), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -346,28 +1759,58 @@ datetime_datetime_now(PyObject *type, PyObject *const *args, Py_ssize_t nargs, P # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"tz", NULL}; + static const char * const _keywords[] = {"sep", "timespec", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "now", + .fname = "isoformat", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; + PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *tz = Py_None; + int sep = 'T'; + const char *timespec = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, - /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } if (!noptargs) { goto skip_optional_pos; } - tz = args[0]; + if (args[0]) { + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("isoformat", "argument 'sep'", "a unicode character", args[0]); + goto exit; + } + if (PyUnicode_GET_LENGTH(args[0]) != 1) { + PyErr_Format(PyExc_TypeError, + "isoformat(): argument 'sep' must be a unicode character, " + "not a string of length %zd", + PyUnicode_GET_LENGTH(args[0])); + goto exit; + } + sep = PyUnicode_READ_CHAR(args[0], 0); + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("isoformat", "argument 'timespec'", "str", args[1]); + goto exit; + } + Py_ssize_t timespec_length; + timespec = PyUnicode_AsUTF8AndSize(args[1], ×pec_length); + if (timespec == NULL) { + goto exit; + } + if (strlen(timespec) != (size_t)timespec_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } skip_optional_pos: - return_value = datetime_datetime_now_impl((PyTypeObject *)type, tz); + return_value = datetime_datetime_isoformat_impl((PyDateTime_DateTime *)self, sep, timespec); exit: return return_value; @@ -524,4 +1967,112 @@ datetime_datetime_replace(PyObject *self, PyObject *const *args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=809640e747529c72 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(datetime_datetime_astimezone__doc__, +"astimezone($self, /, tz=None)\n" +"--\n" +"\n" +"Convert to local time in new timezone tz."); + +#define DATETIME_DATETIME_ASTIMEZONE_METHODDEF \ + {"astimezone", _PyCFunction_CAST(datetime_datetime_astimezone), METH_FASTCALL|METH_KEYWORDS, datetime_datetime_astimezone__doc__}, + +static PyObject * +datetime_datetime_astimezone_impl(PyDateTime_DateTime *self, + PyObject *tzinfo); + +static PyObject * +datetime_datetime_astimezone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(tz), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"tz", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "astimezone", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *tzinfo = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + tzinfo = args[0]; +skip_optional_pos: + return_value = datetime_datetime_astimezone_impl((PyDateTime_DateTime *)self, tzinfo); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime___reduce_ex____doc__, +"__reduce_ex__($self, proto, /)\n" +"--\n" +"\n"); + +#define DATETIME_DATETIME___REDUCE_EX___METHODDEF \ + {"__reduce_ex__", (PyCFunction)datetime_datetime___reduce_ex__, METH_O, datetime_datetime___reduce_ex____doc__}, + +static PyObject * +datetime_datetime___reduce_ex___impl(PyDateTime_DateTime *self, int proto); + +static PyObject * +datetime_datetime___reduce_ex__(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + int proto; + + proto = PyLong_AsInt(arg); + if (proto == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = datetime_datetime___reduce_ex___impl((PyDateTime_DateTime *)self, proto); + +exit: + return return_value; +} + +PyDoc_STRVAR(datetime_datetime___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n"); + +#define DATETIME_DATETIME___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)datetime_datetime___reduce__, METH_NOARGS, datetime_datetime___reduce____doc__}, + +static PyObject * +datetime_datetime___reduce___impl(PyDateTime_DateTime *self); + +static PyObject * +datetime_datetime___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return datetime_datetime___reduce___impl((PyDateTime_DateTime *)self); +} +/*[clinic end generated code: output=0b8403bc58982e60 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/resource.c.h b/Modules/clinic/resource.c.h index 9eda7de27532a1..e4ef93900d1797 100644 --- a/Modules/clinic/resource.c.h +++ b/Modules/clinic/resource.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + #if defined(HAVE_GETRUSAGE) PyDoc_STRVAR(resource_getrusage__doc__, @@ -66,7 +68,7 @@ PyDoc_STRVAR(resource_setrlimit__doc__, "\n"); #define RESOURCE_SETRLIMIT_METHODDEF \ - {"setrlimit", (PyCFunction)(void(*)(void))resource_setrlimit, METH_FASTCALL, resource_setrlimit__doc__}, + {"setrlimit", _PyCFunction_CAST(resource_setrlimit), METH_FASTCALL, resource_setrlimit__doc__}, static PyObject * resource_setrlimit_impl(PyObject *module, int resource, PyObject *limits); @@ -78,8 +80,7 @@ resource_setrlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int resource; PyObject *limits; - if (nargs != 2) { - PyErr_Format(PyExc_TypeError, "setrlimit expected 2 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("setrlimit", nargs, 2, 2)) { goto exit; } resource = PyLong_AsInt(args[0]); @@ -101,7 +102,7 @@ PyDoc_STRVAR(resource_prlimit__doc__, "\n"); #define RESOURCE_PRLIMIT_METHODDEF \ - {"prlimit", (PyCFunction)(void(*)(void))resource_prlimit, METH_FASTCALL, resource_prlimit__doc__}, + {"prlimit", _PyCFunction_CAST(resource_prlimit), METH_FASTCALL, resource_prlimit__doc__}, static PyObject * resource_prlimit_impl(PyObject *module, pid_t pid, int resource, @@ -115,12 +116,7 @@ resource_prlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int resource; PyObject *limits = Py_None; - if (nargs < 2) { - PyErr_Format(PyExc_TypeError, "prlimit expected at least 2 arguments, got %zd", nargs); - goto exit; - } - if (nargs > 3) { - PyErr_Format(PyExc_TypeError, "prlimit expected at most 3 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("prlimit", nargs, 2, 3)) { goto exit; } pid = PyLong_AsPid(args[0]); @@ -178,4 +174,4 @@ resource_getpagesize(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef RESOURCE_PRLIMIT_METHODDEF #define RESOURCE_PRLIMIT_METHODDEF #endif /* !defined(RESOURCE_PRLIMIT_METHODDEF) */ -/*[clinic end generated code: output=e45883ace510414a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8e905b2f5c35170e input=a9049054013a1b77]*/ diff --git a/Modules/gc_weakref.txt b/Modules/gc_weakref.txt index f53fb99dd6cdcb..c3b8cc743ccd21 100644 --- a/Modules/gc_weakref.txt +++ b/Modules/gc_weakref.txt @@ -1,6 +1,16 @@ Intro ===== +************************************************************************** +Note: this document was written long ago, before PEP 442 (safe object +finalization) was implemented. While that has changed some things, this +document is still mostly accurate. Just note that the rules being discussed +here apply to the unreachable set of objects *after* non-legacy finalizers +have been called. Also, the clearing of weakrefs has been changed to happen +later in the collection (after running finalizers but before tp_clear is +called). +************************************************************************** + The basic rule for dealing with weakref callbacks (and __del__ methods too, for that matter) during cyclic gc: diff --git a/Modules/hashlib.h b/Modules/hashlib.h new file mode 100644 index 00000000000000..5ada4ef4b863ec --- /dev/null +++ b/Modules/hashlib.h @@ -0,0 +1,173 @@ +/* Common code for use by all hashlib related modules. */ + +#include "pycore_lock.h" // PyMutex + +/* + * Internal error messages used for reporting an unsupported hash algorithm. + * The algorithm can be given by its name, a callable or a PEP-247 module. + * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() + * and _hmacmodule.c::find_hash_info(). + */ +#define HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" +#define HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" + +/* + * Obtain a buffer view from a buffer-like object 'obj'. + * + * On success, store the result in 'view' and return 0. + * On error, set an exception and return -1. + */ +static inline int +_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view) +{ + if (PyUnicode_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "Strings must be encoded before hashing"); + return -1; + } + if (!PyObject_CheckBuffer(obj)) { + PyErr_SetString(PyExc_TypeError, + "object supporting the buffer API required"); + return -1; + } + if (PyObject_GetBuffer(obj, view, PyBUF_SIMPLE) == -1) { + return -1; + } + if (view->ndim > 1) { + PyErr_SetString(PyExc_BufferError, + "Buffer must be single dimension"); + PyBuffer_Release(view); + return -1; + } + return 0; +} + +/* + * Call _Py_hashlib_get_buffer_view() and check if it succeeded. + * + * On error, set an exception and execute the ERRACTION statements. + */ +#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION) \ + do { \ + if (_Py_hashlib_get_buffer_view(OBJ, VIEW) < 0) { \ + assert(PyErr_Occurred()); \ + ERRACTION; \ + } \ + } while (0) + +#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW) \ + GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL) + +/* + * Helper code to synchronize access to the hash object when the GIL is + * released around a CPU consuming hashlib operation. + * + * Code accessing a mutable part of the hash object must be enclosed in + * an HASHLIB_{ACQUIRE,RELEASE}_LOCK block or explicitly acquire and release + * the mutex inside a Py_BEGIN_ALLOW_THREADS -- Py_END_ALLOW_THREADS block if + * they wish to release the GIL for an operation. + */ + +#define HASHLIB_OBJECT_HEAD \ + PyObject_HEAD \ + /* Guard against race conditions during incremental update(). */ \ + PyMutex mutex; + +#define HASHLIB_INIT_MUTEX(OBJ) \ + do { \ + (OBJ)->mutex = (PyMutex){0}; \ + } while (0) + +#define HASHLIB_ACQUIRE_LOCK(OBJ) PyMutex_Lock(&(OBJ)->mutex) +#define HASHLIB_RELEASE_LOCK(OBJ) PyMutex_Unlock(&(OBJ)->mutex) + +/* + * Message length above which the GIL is to be released + * when performing hashing operations. + */ +#define HASHLIB_GIL_MINSIZE 2048 + +// Macros for executing code while conditionally holding the GIL. +// +// These only drop the GIL if the lock acquisition itself is likely to +// block. Thus the non-blocking acquire gating the GIL release for a +// blocking lock acquisition. The intent of these macros is to surround +// the assumed always "fast" operations that you aren't releasing the +// GIL around. + +/* + * Execute a suite of C statements 'STATEMENTS'. + * + * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. + */ +#define HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(SIZE, STATEMENTS) \ + do { \ + if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + STATEMENTS; \ + Py_END_ALLOW_THREADS \ + } \ + else { \ + STATEMENTS; \ + } \ + } while (0) + +/* + * Lock 'OBJ' and execute a suite of C statements 'STATEMENTS'. + * + * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. + */ +#define HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(OBJ, SIZE, STATEMENTS) \ + do { \ + if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + HASHLIB_ACQUIRE_LOCK(OBJ); \ + STATEMENTS; \ + HASHLIB_RELEASE_LOCK(OBJ); \ + Py_END_ALLOW_THREADS \ + } \ + else { \ + HASHLIB_ACQUIRE_LOCK(OBJ); \ + STATEMENTS; \ + HASHLIB_RELEASE_LOCK(OBJ); \ + } \ + } while (0) + +static inline int +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) +{ + if (data != NULL && string == NULL) { + // called as H(data) or H(data=...) + *res = data; + return 1; + } + else if (data == NULL && string != NULL) { + // called as H(string=...) + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead", 1) < 0) + { + *res = NULL; + return -1; + } + *res = string; + return 1; + } + else if (data == NULL && string == NULL) { + // fast path when no data is given + assert(!PyErr_Occurred()); + *res = NULL; + return 0; + } + else { + // called as H(data=..., string) + *res = NULL; + PyErr_SetString(PyExc_TypeError, + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version."); + return -1; + } +} diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index 92be49c5a879f3..694e2a095744ff 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -20,10 +20,6 @@ #include "pycore_hashtable.h" #include "pycore_strhex.h" // _Py_strhex() -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_fetch.h" -#include "_hashlib/hashlib_mutex.h" - /* * Taken from blake2module.c. In the future, detection of SIMD support * should be delegated to https://github.com/python/cpython/pull/125011. @@ -51,6 +47,8 @@ #include +#include "hashlib.h" + // --- Reusable error messages ------------------------------------------------ static inline void @@ -658,7 +656,7 @@ find_hash_info(hmacmodule_state *state, PyObject *hash_info_ref) } if (rc == 0) { PyErr_Format(state->unknown_hash_error, - _Py_HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); + HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); return NULL; } assert(info != NULL); diff --git a/Modules/md5module.c b/Modules/md5module.c index d5dc4f60a575d4..8b6dd4a8195dfb 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -22,10 +22,9 @@ #endif #include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_strhex.h" // _Py_strhex() -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_mutex.h" +#include "hashlib.h" #include "_hacl/Hacl_Hash_MD5.h" diff --git a/Modules/resource.c b/Modules/resource.c index 3fe18e7c98e3d8..2353bc6653abd8 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,7 +1,5 @@ -// Need limited C API version 3.13 for PySys_Audit() -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" @@ -150,6 +148,35 @@ resource_getrusage_impl(PyObject *module, int who) } #endif +static int +py2rlim(PyObject *obj, rlim_t *out) +{ + obj = PyNumber_Index(obj); + if (obj == NULL) { + return -1; + } + int neg = PyLong_IsNegative(obj); + assert(neg >= 0); + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, out, sizeof(*out), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + Py_DECREF(obj); + if (bytes < 0) { + return -1; + } + else if (neg && (*out != RLIM_INFINITY || bytes > (Py_ssize_t)sizeof(*out))) { + PyErr_SetString(PyExc_ValueError, + "Cannot convert negative int"); + return -1; + } + else if (bytes > (Py_ssize_t)sizeof(*out)) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C rlim_t"); + return -1; + } + return 0; +} + static int py2rlimit(PyObject *limits, struct rlimit *rl_out) { @@ -166,26 +193,13 @@ py2rlimit(PyObject *limits, struct rlimit *rl_out) } curobj = PyTuple_GetItem(limits, 0); // borrowed maxobj = PyTuple_GetItem(limits, 1); // borrowed -#if !defined(HAVE_LARGEFILE_SUPPORT) - rl_out->rlim_cur = PyLong_AsLong(curobj); - if (rl_out->rlim_cur == (rlim_t)-1 && PyErr_Occurred()) - goto error; - rl_out->rlim_max = PyLong_AsLong(maxobj); - if (rl_out->rlim_max == (rlim_t)-1 && PyErr_Occurred()) - goto error; -#else - /* The limits are probably bigger than a long */ - rl_out->rlim_cur = PyLong_AsLongLong(curobj); - if (rl_out->rlim_cur == (rlim_t)-1 && PyErr_Occurred()) - goto error; - rl_out->rlim_max = PyLong_AsLongLong(maxobj); - if (rl_out->rlim_max == (rlim_t)-1 && PyErr_Occurred()) + if (py2rlim(curobj, &rl_out->rlim_cur) < 0 || + py2rlim(maxobj, &rl_out->rlim_max) < 0) + { goto error; -#endif + } Py_DECREF(limits); - rl_out->rlim_cur = rl_out->rlim_cur & RLIM_INFINITY; - rl_out->rlim_max = rl_out->rlim_max & RLIM_INFINITY; return 0; error: @@ -193,15 +207,24 @@ py2rlimit(PyObject *limits, struct rlimit *rl_out) return -1; } +static PyObject* +rlim2py(rlim_t value) +{ + if (value == RLIM_INFINITY) { + return PyLong_FromNativeBytes(&value, sizeof(value), -1); + } + return PyLong_FromUnsignedNativeBytes(&value, sizeof(value), -1); +} + static PyObject* rlimit2py(struct rlimit rl) { - if (sizeof(rl.rlim_cur) > sizeof(long)) { - return Py_BuildValue("LL", - (long long) rl.rlim_cur, - (long long) rl.rlim_max); + PyObject *cur = rlim2py(rl.rlim_cur); + if (cur == NULL) { + return NULL; } - return Py_BuildValue("ll", (long) rl.rlim_cur, (long) rl.rlim_max); + PyObject *max = rlim2py(rl.rlim_max); + return Py_BuildValue("NN", cur, max); } /*[clinic input] @@ -495,14 +518,7 @@ resource_exec(PyObject *module) ADD_INT(module, RLIMIT_KQUEUES); #endif - PyObject *v; - if (sizeof(RLIM_INFINITY) > sizeof(long)) { - v = PyLong_FromLongLong((long long) RLIM_INFINITY); - } else - { - v = PyLong_FromLong((long) RLIM_INFINITY); - } - if (PyModule_Add(module, "RLIM_INFINITY", v) < 0) { + if (PyModule_Add(module, "RLIM_INFINITY", rlim2py(RLIM_INFINITY)) < 0) { return -1; } return 0; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index 86e5691e8463e4..faa9dcccc5755b 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -20,11 +20,9 @@ #endif #include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" // _PyType_GetModuleState() - -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_mutex.h" +#include "hashlib.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "_hacl/Hacl_Hash_SHA1.h" diff --git a/Modules/sha2module.c b/Modules/sha2module.c index dbf6dde1b8c121..36300ba899fd44 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -21,12 +21,11 @@ #endif #include "Python.h" -#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_strhex.h" // _Py_strhex() -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_mutex.h" +#include "hashlib.h" #include "_hacl/Hacl_Hash_SHA2.h" diff --git a/Modules/sha3module.c b/Modules/sha3module.c index c67bfadbe4664a..5764556bb680f3 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -21,11 +21,9 @@ #endif #include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" // _PyType_GetModuleState() - -#include "_hashlib/hashlib_buffer.h" -#include "_hashlib/hashlib_mutex.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "hashlib.h" #include "_hacl/Hacl_Hash_SHA3.h" diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f3ad01854de93b..bca9e7bb712e38 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -3332,32 +3332,59 @@ sock_setsockopt(PyObject *self, PyObject *args) { PySocketSockObject *s = _PySocketSockObject_CAST(self); + Py_ssize_t arglen; int level; int optname; int res; - Py_buffer optval; + Py_buffer buffer; int flag; unsigned int optlen; - PyObject *none; + PyObject *optval; + + if (!PyArg_ParseTuple(args, "iiO|I:setsockopt", + &level, &optname, &optval, &optlen)) + { + return NULL; + } + + arglen = PyTuple_Size(args); + if (arglen == 3 && optval == Py_None) { + PyErr_Format(PyExc_TypeError, + "setsockopt() requires 4 arguments when the third argument is None", + arglen); + return NULL; + } + if (arglen == 4 && optval != Py_None) { + PyErr_Format(PyExc_TypeError, + "setsockopt() only takes 4 arguments when the third argument is None (got %T)", + optval); + return NULL; + } #ifdef AF_VSOCK if (s->sock_family == AF_VSOCK) { + if (!PyIndex_Check(optval)) { + PyErr_Format(PyExc_TypeError, + "setsockopt() argument 3 for AF_VSOCK must be an int (got %T)", + optval); + } uint64_t vflag; // Must be set width of 64 bits /* setsockopt(level, opt, flag) */ - if (PyArg_ParseTuple(args, "iiK:setsockopt", - &level, &optname, &vflag)) { - // level should always be set to AF_VSOCK - res = setsockopt(get_sock_fd(s), level, optname, - (void*)&vflag, sizeof vflag); - goto done; + if (!PyArg_Parse(optval, "K", &vflag)) { + return NULL; } - return NULL; + // level should always be set to AF_VSOCK + res = setsockopt(get_sock_fd(s), level, optname, + (void*)&vflag, sizeof vflag); + goto done; } #endif /* setsockopt(level, opt, flag) */ - if (PyArg_ParseTuple(args, "iii:setsockopt", - &level, &optname, &flag)) { + if (PyIndex_Check(optval)) { + if (!PyArg_Parse(optval, "i", &flag)) { + return NULL; + } #ifdef MS_WINDOWS if (optname == SIO_TCP_SET_ACK_FREQUENCY) { DWORD dummy; @@ -3374,36 +3401,40 @@ sock_setsockopt(PyObject *self, PyObject *args) goto done; } - PyErr_Clear(); - /* setsockopt(level, opt, None, flag) */ - if (PyArg_ParseTuple(args, "iiO!I:setsockopt", - &level, &optname, Py_TYPE(Py_None), &none, &optlen)) { + /* setsockopt(level, opt, None, optlen) */ + if (optval == Py_None) { assert(sizeof(socklen_t) >= sizeof(unsigned int)); res = setsockopt(get_sock_fd(s), level, optname, NULL, (socklen_t)optlen); goto done; } - PyErr_Clear(); /* setsockopt(level, opt, buffer) */ - if (!PyArg_ParseTuple(args, "iiy*:setsockopt", - &level, &optname, &optval)) - return NULL; - + if (PyObject_CheckBuffer(optval)) { + if (!PyArg_Parse(optval, "y*", &buffer)) { + return NULL; + } #ifdef MS_WINDOWS - if (optval.len > INT_MAX) { - PyBuffer_Release(&optval); - PyErr_Format(PyExc_OverflowError, - "socket option is larger than %i bytes", - INT_MAX); - return NULL; - } - res = setsockopt(get_sock_fd(s), level, optname, - optval.buf, (int)optval.len); + if (buffer.len > INT_MAX) { + PyBuffer_Release(&buffer); + PyErr_Format(PyExc_OverflowError, + "socket option is larger than %i bytes", + INT_MAX); + return NULL; + } + res = setsockopt(get_sock_fd(s), level, optname, + buffer.buf, (int)buffer.len); #else - res = setsockopt(get_sock_fd(s), level, optname, optval.buf, optval.len); + res = setsockopt(get_sock_fd(s), level, optname, buffer.buf, buffer.len); #endif - PyBuffer_Release(&optval); + PyBuffer_Release(&buffer); + goto done; + } + + PyErr_Format(PyExc_TypeError, + "socket option should be int, bytes-like object or None (got %T)", + optval); + return NULL; done: if (res < 0) { diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 42e021679b583f..478c571345cd03 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -550,16 +550,12 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&interp->func_state.mutex); -#endif + FT_MUTEX_LOCK(&interp->func_state.mutex); co->co_version = interp->func_state.next_version; if (interp->func_state.next_version != 0) { interp->func_state.next_version++; } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&interp->func_state.mutex); -#endif + FT_MUTEX_UNLOCK(&interp->func_state.mutex); co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; /* not set */ @@ -689,7 +685,7 @@ intern_code_constants(struct _PyCodeConstructor *con) #ifdef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); struct _py_code_state *state = &interp->code_state; - PyMutex_Lock(&state->mutex); + FT_MUTEX_LOCK(&state->mutex); #endif if (intern_strings(con->names) < 0) { goto error; @@ -700,15 +696,11 @@ intern_code_constants(struct _PyCodeConstructor *con) if (intern_strings(con->localsplusnames) < 0) { goto error; } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&state->mutex); -#endif + FT_MUTEX_UNLOCK(&state->mutex); return 0; error: -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&state->mutex); -#endif + FT_MUTEX_UNLOCK(&state->mutex); return -1; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 97de1e06efe1b2..72c0ab0666e927 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1685,6 +1685,8 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value) case PY_MONITORING_EVENT_PY_RESUME: case PY_MONITORING_EVENT_JUMP: case PY_MONITORING_EVENT_BRANCH: + case PY_MONITORING_EVENT_BRANCH_LEFT: + case PY_MONITORING_EVENT_BRANCH_RIGHT: case PY_MONITORING_EVENT_LINE: case PY_MONITORING_EVENT_PY_YIELD: /* Setting f_lineno is allowed for the above events */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index d952a58d94af55..14bc5a4bc49f84 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1535,9 +1535,13 @@ type_set_name(PyObject *tp, PyObject *value, void *Py_UNUSED(closure)) return -1; } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); type->tp_name = tp_name; - Py_SETREF(((PyHeapTypeObject*)type)->ht_name, Py_NewRef(value)); - + PyObject *old_name = ((PyHeapTypeObject*)type)->ht_name; + ((PyHeapTypeObject*)type)->ht_name = Py_NewRef(value); + _PyEval_StartTheWorld(interp); + Py_DECREF(old_name); return 0; } @@ -10581,7 +10585,10 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) getattr = _PyType_LookupRef(tp, &_Py_ID(__getattr__)); if (getattr == NULL) { /* No __getattr__ hook: use a simpler dispatcher */ +#ifndef Py_GIL_DISABLED + // Replacing the slot is only thread-safe if there is a GIL. tp->tp_getattro = _Py_slot_tp_getattro; +#endif return _Py_slot_tp_getattro(self, name); } /* speed hack: we could use lookup_maybe, but that would resolve the @@ -10706,9 +10713,11 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) get = _PyType_LookupRef(tp, &_Py_ID(__get__)); if (get == NULL) { +#ifndef Py_GIL_DISABLED /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) tp->tp_descr_get = NULL; +#endif return Py_NewRef(self); } if (obj == NULL) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 8df7a48284dccd..47802c5e915be2 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -114,14 +114,6 @@ NOTE: In the interpreter's initialization phase, some globals are currently # define _PyUnicode_CHECK(op) PyUnicode_Check(op) #endif -#ifdef Py_GIL_DISABLED -# define LOCK_INTERNED(interp) PyMutex_Lock(&_Py_INTERP_CACHED_OBJECT(interp, interned_mutex)) -# define UNLOCK_INTERNED(interp) PyMutex_Unlock(&_Py_INTERP_CACHED_OBJECT(interp, interned_mutex)) -#else -# define LOCK_INTERNED(interp) -# define UNLOCK_INTERNED(interp) -#endif - static inline char* _PyUnicode_UTF8(PyObject *op) { return FT_ATOMIC_LOAD_PTR_ACQUIRE(_PyCompactUnicodeObject_CAST(op)->utf8); @@ -7684,10 +7676,6 @@ code_page_name(UINT code_page, PyObject **obj) *obj = NULL; if (code_page == CP_ACP) return "mbcs"; - if (code_page == CP_UTF7) - return "CP_UTF7"; - if (code_page == CP_UTF8) - return "CP_UTF8"; *obj = PyBytes_FromFormat("cp%u", code_page); if (*obj == NULL) @@ -15992,14 +15980,16 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, /* Do a setdefault on the per-interpreter cache. */ PyObject *interned = get_interned_dict(interp); assert(interned != NULL); - - LOCK_INTERNED(interp); +#ifdef Py_GIL_DISABLED +# define INTERN_MUTEX &_Py_INTERP_CACHED_OBJECT(interp, interned_mutex) +#endif + FT_MUTEX_LOCK(INTERN_MUTEX); PyObject *t; { int res = PyDict_SetDefaultRef(interned, s, s, &t); if (res < 0) { PyErr_Clear(); - UNLOCK_INTERNED(interp); + FT_MUTEX_UNLOCK(INTERN_MUTEX); return s; } else if (res == 1) { @@ -16009,7 +15999,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, PyUnicode_CHECK_INTERNED(t) == SSTATE_INTERNED_MORTAL) { immortalize_interned(t); } - UNLOCK_INTERNED(interp); + FT_MUTEX_UNLOCK(INTERN_MUTEX); return t; } else { @@ -16042,7 +16032,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, immortalize_interned(s); } - UNLOCK_INTERNED(interp); + FT_MUTEX_UNLOCK(INTERN_MUTEX); return s; } diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index cfb43cee935b86..2cd205224bc089 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -100,11 +100,6 @@ - - - - - diff --git a/PCbuild/_hashlib.vcxproj.filters b/PCbuild/_hashlib.vcxproj.filters index d465d92a956eda..7a0700c007f644 100644 --- a/PCbuild/_hashlib.vcxproj.filters +++ b/PCbuild/_hashlib.vcxproj.filters @@ -18,4 +18,4 @@ Resource Files - + \ No newline at end of file diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index e29054f5734d49..eff8d1ccd7f146 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -56,7 +56,7 @@ set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.16 set libraries=%libraries% mpdecimal-4.0.0 -set libraries=%libraries% sqlite-3.49.1.0 +set libraries=%libraries% sqlite-3.50.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 set libraries=%libraries% xz-5.2.5 diff --git a/PCbuild/python.props b/PCbuild/python.props index ddc7696d2762fe..e1c2ff3fe3cc11 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -74,7 +74,7 @@ - $(ExternalsDir)sqlite-3.49.1.0\ + $(ExternalsDir)sqlite-3.50.4.0\ $(ExternalsDir)bzip2-1.0.8\ $(ExternalsDir)xz-5.2.5\ $(ExternalsDir)libffi-3.4.4\ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c59b380d814ed9..517103acea8d8e 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -447,10 +447,6 @@ HACL_CAN_COMPILE_VEC128;%(PreprocessorDefinitions) /arch:AVX %(AdditionalOptions) - - - - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 1410cbbef6c849..e9eedfd1312fae 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -255,15 +255,6 @@ Include - - Modules\_hashlib - - - Modules\_hashlib - - - Modules\_hashlib - Modules @@ -980,9 +971,6 @@ Modules - - Modules\_hashlib - Modules diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 3ae3255d933967..27c0d382281bdb 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -237,7 +237,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.49.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.50.4, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index ebc94715b6f361..0763866576733f 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1404,7 +1404,15 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) { if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) { return NULL; } - PyObject* str = _PyPegen_decode_string(p, 0, bstr, bsize, tok); + + // Check if we're inside a raw f-string for format spec decoding + int is_raw = 0; + if (INSIDE_FSTRING(p->tok)) { + tokenizer_mode *mode = TOK_GET_MODE(p->tok); + is_raw = mode->raw; + } + + PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok); if (str == NULL) { return NULL; } diff --git a/Parser/pegen.c b/Parser/pegen.c index 50641de27d37fd..70493031656028 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -5,6 +5,7 @@ #include "pycore_pyerrors.h" // PyExc_IncompleteInputError #include "pycore_runtime.h" // _PyRuntime #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal +#include "pycore_pyatomic_ft_wrappers.h" #include #include "lexer/lexer.h" @@ -299,22 +300,14 @@ _PyPegen_fill_token(Parser *p) #define NSTATISTICS _PYPEGEN_NSTATISTICS #define memo_statistics _PyRuntime.parser.memo_statistics -#ifdef Py_GIL_DISABLED -#define MUTEX_LOCK() PyMutex_Lock(&_PyRuntime.parser.mutex) -#define MUTEX_UNLOCK() PyMutex_Unlock(&_PyRuntime.parser.mutex) -#else -#define MUTEX_LOCK() -#define MUTEX_UNLOCK() -#endif - void _PyPegen_clear_memo_statistics(void) { - MUTEX_LOCK(); + FT_MUTEX_LOCK(&_PyRuntime.parser.mutex); for (int i = 0; i < NSTATISTICS; i++) { memo_statistics[i] = 0; } - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); } PyObject * @@ -325,22 +318,22 @@ _PyPegen_get_memo_statistics(void) return NULL; } - MUTEX_LOCK(); + FT_MUTEX_LOCK(&_PyRuntime.parser.mutex); for (int i = 0; i < NSTATISTICS; i++) { PyObject *value = PyLong_FromLong(memo_statistics[i]); if (value == NULL) { - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); Py_DECREF(ret); return NULL; } // PyList_SetItem borrows a reference to value. if (PyList_SetItem(ret, i, value) < 0) { - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); Py_DECREF(ret); return NULL; } } - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); return ret; } #endif @@ -366,9 +359,9 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) if (count <= 0) { count = 1; } - MUTEX_LOCK(); + FT_MUTEX_LOCK(&_PyRuntime.parser.mutex); memo_statistics[type] += count; - MUTEX_UNLOCK(); + FT_MUTEX_UNLOCK(&_PyRuntime.parser.mutex); } #endif p->mark = m->mark; diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 06d1ee016dc2a8..a5809b37b6b493 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -45,27 +45,40 @@ static const char header[] = static void runtime_init(void) { - PyConfig config; - PyConfig_InitIsolatedConfig(&config); - - config.site_import = 0; + PyInitConfig *config = PyInitConfig_Create(); + if (config == NULL) { + printf("memory allocation failed\n"); + exit(1); + } - PyStatus status; - status = PyConfig_SetString(&config, &config.program_name, - L"./_freeze_module"); - if (PyStatus_Exception(status)) { - PyConfig_Clear(&config); - Py_ExitStatusException(status); + if (PyInitConfig_SetInt(config, "site_import", 0) < 0) { + goto error; + } + if (PyInitConfig_SetStr(config, "program_name", "./_freeze_module") < 0) { + goto error; } /* Don't install importlib, since it could execute outdated bytecode. */ - config._install_importlib = 0; - config._init_main = 0; + if (PyInitConfig_SetInt(config, "_install_importlib", 0) < 0) { + goto error; + } + if (PyInitConfig_SetInt(config, "_init_main", 0) < 0) { + goto error; + } + + if (Py_InitializeFromInitConfig(config) < 0) { + goto error; + } + PyInitConfig_Free(config); + return; - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); +error: + { + const char *err_msg; + (void)PyInitConfig_GetError(config, &err_msg); + printf("Python init error: %s\n", err_msg); + PyInitConfig_Free(config); + exit(1); } } diff --git a/Python/ast_preprocess.c b/Python/ast_preprocess.c index bafd67ed790b20..44d3075098be75 100644 --- a/Python/ast_preprocess.c +++ b/Python/ast_preprocess.c @@ -435,13 +435,38 @@ stmt_seq_remove_item(asdl_stmt_seq *stmts, Py_ssize_t idx) return 1; } +static int +remove_docstring(asdl_stmt_seq *stmts, Py_ssize_t idx, PyArena *ctx_) +{ + assert(_PyAST_GetDocString(stmts) != NULL); + // In case there's just the docstring in the body, replace it with `pass` + // keyword, so body won't be empty. + if (asdl_seq_LEN(stmts) == 1) { + stmt_ty docstring = (stmt_ty)asdl_seq_GET(stmts, 0); + stmt_ty pass = _PyAST_Pass( + docstring->lineno, docstring->col_offset, + // we know that `pass` always takes 4 chars and a single line, + // while docstring can span on multiple lines + docstring->lineno, docstring->col_offset + 4, + ctx_ + ); + if (pass == NULL) { + return 0; + } + asdl_seq_SET(stmts, 0, pass); + return 1; + } + // In case there are more than 1 body items, just remove the docstring. + return stmt_seq_remove_item(stmts, idx); +} + static int astfold_body(asdl_stmt_seq *stmts, PyArena *ctx_, _PyASTPreprocessState *state) { int docstring = _PyAST_GetDocString(stmts) != NULL; if (docstring && (state->optimize >= 2)) { /* remove the docstring */ - if (!stmt_seq_remove_item(stmts, 0)) { + if (!remove_docstring(stmts, 0, ctx_)) { return 0; } docstring = 0; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d9abc4c53d1f50..4d6dbe5116626d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -153,22 +153,19 @@ dummy_func( macro(NOT_TAKEN) = NOP; op(_CHECK_PERIODIC, (--)) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - int err = _Py_HandlePending(tstate); - ERROR_IF(err != 0); - } + int err = check_periodics(tstate); + ERROR_IF(err != 0); + } + + replaced op(_CHECK_PERIODIC_AT_END, (--)) { + int err = check_periodics(tstate); + ERROR_IF(err != 0); } op(_CHECK_PERIODIC_IF_NOT_YIELD_FROM, (--)) { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - int err = _Py_HandlePending(tstate); - ERROR_IF(err != 0); - } + int err = check_periodics(tstate); + ERROR_IF(err != 0); } } @@ -2964,6 +2961,8 @@ dummy_func( else { this_instr[1].counter = initial_jump_backoff_counter(); assert(tstate->current_executor == NULL); + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; GOTO_TIER_TWO(executor); } } @@ -3028,6 +3027,8 @@ dummy_func( } DISPATCH_GOTO(); } + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; GOTO_TIER_TWO(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -3790,8 +3791,8 @@ dummy_func( ERROR_IF(err); } - macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC; - macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC; + macro(CALL) = _SPECIALIZE_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC_AT_END; + macro(INSTRUMENTED_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC_AT_END; op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -3912,7 +3913,7 @@ dummy_func( unused/2 + _CHECK_IS_NOT_PY_CALLABLE + _CALL_NON_PY_GENERAL + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { EXIT_IF(!PyStackRef_IsNull(null)); @@ -4069,7 +4070,7 @@ dummy_func( _GUARD_NOS_NULL + _GUARD_CALLABLE_STR_1 + _CALL_STR_1 + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_GUARD_CALLABLE_TUPLE_1, (callable, unused, unused -- callable, unused, unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4097,7 +4098,7 @@ dummy_func( _GUARD_NOS_NULL + _GUARD_CALLABLE_TUPLE_1 + _CALL_TUPLE_1 + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4193,7 +4194,7 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_CLASS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_O functions */ @@ -4228,7 +4229,7 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_O + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ @@ -4265,7 +4266,7 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_FAST + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ @@ -4301,7 +4302,7 @@ dummy_func( unused/1 + unused/2 + _CALL_BUILTIN_FAST_WITH_KEYWORDS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; macro(CALL_LEN) = unused/1 + @@ -4439,7 +4440,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_O + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4481,7 +4482,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_METHOD_DESCRIPTOR_NOARGS, (callable, self_or_null, args[oparg] -- res)) { assert(oparg == 0 || oparg == 1); @@ -4519,7 +4520,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_NOARGS + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_CALL_METHOD_DESCRIPTOR_FAST, (callable, self_or_null, args[oparg] -- res)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4560,7 +4561,7 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_FAST + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; // Cache layout: counter/1, func_version/2 family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = { @@ -4816,7 +4817,7 @@ dummy_func( unused/2 + _CHECK_IS_NOT_PY_CALLABLE_KW + _CALL_KW_NON_PY + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; op(_MAKE_CALLARGS_A_TUPLE, (func, unused, callargs, kwargs -- func, unused, callargs, kwargs)) { PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); @@ -4917,12 +4918,12 @@ dummy_func( macro(CALL_FUNCTION_EX) = _MAKE_CALLARGS_A_TUPLE + _DO_CALL_FUNCTION_EX + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; macro(INSTRUMENTED_CALL_FUNCTION_EX) = _MAKE_CALLARGS_A_TUPLE + _DO_CALL_FUNCTION_EX + - _CHECK_PERIODIC; + _CHECK_PERIODIC_AT_END; inst(MAKE_FUNCTION, (codeobj_st -- func)) { PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); @@ -5185,23 +5186,20 @@ dummy_func( op (_GUARD_IS_TRUE_POP, (flag -- )) { int is_true = PyStackRef_IsTrue(flag); DEAD(flag); - SYNC_SP(); - EXIT_IF(!is_true); + AT_END_EXIT_IF(!is_true); } op (_GUARD_IS_FALSE_POP, (flag -- )) { int is_false = PyStackRef_IsFalse(flag); DEAD(flag); - SYNC_SP(); - EXIT_IF(!is_false); + AT_END_EXIT_IF(!is_false); } op (_GUARD_IS_NONE_POP, (val -- )) { int is_none = PyStackRef_IsNone(val); if (!is_none) { PyStackRef_CLOSE(val); - SYNC_SP(); - EXIT_IF(1); + AT_END_EXIT_IF(1); } DEAD(val); } @@ -5209,8 +5207,7 @@ dummy_func( op (_GUARD_IS_NOT_NONE_POP, (val -- )) { int is_none = PyStackRef_IsNone(val); PyStackRef_CLOSE(val); - SYNC_SP(); - EXIT_IF(is_none); + AT_END_EXIT_IF(is_none); } op(_JUMP_TO_TOP, (--)) { @@ -5238,9 +5235,8 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; - PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { printf("SIDE EXIT: [UOp "); @@ -5251,32 +5247,7 @@ dummy_func( _PyOpcode_OpName[target->op.code]); } #endif - if (exit->executor && !exit->executor->vm_data.valid) { - exit->temperature = initial_temperature_backoff_counter(); - Py_CLEAR(exit->executor); - } - if (exit->executor == NULL) { - _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - } - else { - int chain_depth = current_executor->vm_data.chain_depth + 1; - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } - exit->temperature = initial_temperature_backoff_counter(); - } - exit->executor = executor; - } + tstate->jit_exit = exit; GOTO_TIER_TWO(exit->executor); } @@ -5375,7 +5346,14 @@ dummy_func( #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif - assert(((_PyExecutorObject *)executor)->vm_data.valid); + assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); + tstate->current_executor = (PyObject *)executor; + if (!current_executor->vm_data.valid) { + assert(tstate->jit_exit->executor == current_executor); + assert(tstate->current_executor == executor); + _PyExecutor_ClearExit(tstate->jit_exit); + DEOPT_IF(true); + } } tier2 op(_MAKE_WARM, (--)) { @@ -5395,6 +5373,11 @@ dummy_func( GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } + tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { + int err = _Py_HandlePending(tstate); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + } + tier2 op(_ERROR_POP_N, (target/2 --)) { assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; @@ -5406,14 +5389,45 @@ dummy_func( * ENTER_EXECUTOR will not re-enter tier 2 with the eval breaker set. */ tier2 op(_TIER2_RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0); + HANDLE_PENDING_AND_DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); + HANDLE_PENDING_AND_DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } + tier2 op(_COLD_EXIT, ( -- )) { + _PyExitData *exit = tstate->jit_exit; + assert(exit != NULL); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + 1; + int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(temperature); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); + } + exit->temperature = initial_temperature_backoff_counter(); + } + assert(tstate->jit_exit == exit); + exit->executor = executor; + GOTO_TIER_TWO(exit->executor); + } + label(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/ceval.c b/Python/ceval.c index 9ccd42bdf0a55c..94c24187f6b25f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1158,7 +1158,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uint64_t trace_uop_execution_counter = 0; #endif - assert(next_uop->opcode == _START_EXECUTOR); + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); tier2_dispatch: for (;;) { uopcode = next_uop->opcode; @@ -1211,6 +1211,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int printf(" @ %d -> %s]\n", (int)(next_uop - current_executor->trace - 1), _PyOpcode_OpName[frame->instr_ptr->op.code]); + fflush(stdout); } #endif assert(next_uop[-1].format == UOP_FORMAT_JUMP); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index aa68371ac8febf..6bf64868cbb2d3 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -907,13 +907,9 @@ unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) static void clear_pending_handling_thread(struct _pending_calls *pending) { -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&pending->mutex); - pending->handling_thread = NULL; - PyMutex_Unlock(&pending->mutex); -#else + FT_MUTEX_LOCK(&pending->mutex); pending->handling_thread = NULL; -#endif + FT_MUTEX_UNLOCK(&pending->mutex); } static int diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 187ec8fdd26584..ddbcf2aa1bab02 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -359,7 +359,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer) do { \ OPT_STAT_INC(traces_executed); \ _PyExecutorObject *_executor = (EXECUTOR); \ - tstate->current_executor = (PyObject *)_executor; \ jit_func jitted = _executor->jit_code; \ /* Keep the shim frame alive via the executor: */ \ Py_INCREF(_executor); \ @@ -378,9 +377,8 @@ do { \ do { \ OPT_STAT_INC(traces_executed); \ _PyExecutorObject *_executor = (EXECUTOR); \ - tstate->current_executor = (PyObject *)_executor; \ next_uop = _executor->trace; \ - assert(next_uop->opcode == _START_EXECUTOR); \ + assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); \ goto enter_tier_two; \ } while (0) #endif @@ -390,7 +388,6 @@ do { \ { \ tstate->current_executor = NULL; \ next_instr = (TARGET); \ - assert(tstate->current_executor == NULL); \ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ @@ -425,3 +422,14 @@ do { \ _PyObjectArray_Free(NAME - 1, NAME##_temp); #define CONVERSION_FAILED(NAME) ((NAME) == NULL) + +static inline int +check_periodics(PyThreadState *tstate) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + return _Py_HandlePending(tstate); + } + return 0; +} + diff --git a/Python/codecs.c b/Python/codecs.c index caf8d9d5f3c188..8eb9f2db41359e 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -16,7 +16,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "pycore_runtime.h" // _Py_ID() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "pycore_unicodeobject.h" // _PyUnicode_InternMortal() - +#include "pycore_pyatomic_ft_wrappers.h" static const char *codecs_builtin_error_handlers[] = { "strict", "ignore", "replace", @@ -40,13 +40,10 @@ int PyCodec_Register(PyObject *search_function) PyErr_SetString(PyExc_TypeError, "argument must be callable"); goto onError; } -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_LOCK(&interp->codecs.search_path_mutex); int ret = PyList_Append(interp->codecs.search_path, search_function); -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_UNLOCK(&interp->codecs.search_path_mutex); + return ret; onError: @@ -66,9 +63,7 @@ PyCodec_Unregister(PyObject *search_function) PyObject *codec_search_path = interp->codecs.search_path; assert(PyList_CheckExact(codec_search_path)); for (Py_ssize_t i = 0; i < PyList_GET_SIZE(codec_search_path); i++) { -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_LOCK(&interp->codecs.search_path_mutex); PyObject *item = PyList_GetItemRef(codec_search_path, i); int ret = 1; if (item == search_function) { @@ -76,9 +71,7 @@ PyCodec_Unregister(PyObject *search_function) // while we hold search_path_mutex. ret = PyList_SetSlice(codec_search_path, i, i+1, NULL); } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&interp->codecs.search_path_mutex); -#endif + FT_MUTEX_UNLOCK(&interp->codecs.search_path_mutex); Py_DECREF(item); if (ret != 1) { assert(interp->codecs.search_cache != NULL); @@ -1204,7 +1197,7 @@ get_standard_encoding_impl(const char *encoding, int *bytelength) } } } - else if (strcmp(encoding, "CP_UTF8") == 0) { + else if (strcmp(encoding, "cp65001") == 0) { *bytelength = 3; return ENC_UTF8; } diff --git a/Python/errors.c b/Python/errors.c index a3122f76bdd87d..2688396004e98b 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1444,12 +1444,16 @@ make_unraisable_hook_args(PyThreadState *tstate, PyObject *exc_type, It can be called to log the exception of a custom sys.unraisablehook. - Do nothing if sys.stderr attribute doesn't exist or is set to None. */ + This assumes 'file' is neither NULL nor None. + */ static int write_unraisable_exc_file(PyThreadState *tstate, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb, PyObject *err_msg, PyObject *obj, PyObject *file) { + assert(file != NULL); + assert(!Py_IsNone(file)); + if (obj != NULL && obj != Py_None) { if (err_msg != NULL && err_msg != Py_None) { if (PyFile_WriteObject(err_msg, file, Py_PRINT_RAW) < 0) { @@ -1484,6 +1488,27 @@ write_unraisable_exc_file(PyThreadState *tstate, PyObject *exc_type, } } + // Try printing the exception using the stdlib module. + // If this fails, then we have to use the C implementation. + PyObject *print_exception_fn = PyImport_ImportModuleAttrString("traceback", + "_print_exception_bltin"); + if (print_exception_fn != NULL && PyCallable_Check(print_exception_fn)) { + PyObject *args[2] = {exc_value, file}; + PyObject *result = PyObject_Vectorcall(print_exception_fn, args, 2, NULL); + int ok = (result != NULL); + Py_DECREF(print_exception_fn); + Py_XDECREF(result); + if (ok) { + // Nothing else to do + return 0; + } + } + else { + Py_XDECREF(print_exception_fn); + } + // traceback module failed, fall back to pure C + _PyErr_Clear(tstate); + if (exc_tb != NULL && exc_tb != Py_None) { if (PyTraceBack_Print(exc_tb, file) < 0) { /* continue even if writing the traceback failed */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e152865e4ec9e8..182289922c7637 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -13,31 +13,25 @@ } case _CHECK_PERIODIC: { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_ERROR(); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_ERROR(); } break; } + /* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 because it is replaced */ + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { oparg = CURRENT_OPARG(); if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_ERROR(); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_ERROR(); } } break; @@ -7113,9 +7107,8 @@ case _EXIT_TRACE: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; - PyCodeObject *code = _PyFrame_GetCode(frame); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; #if defined(Py_DEBUG) && !defined(_Py_JIT) + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7128,36 +7121,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } #endif - if (exit->executor && !exit->executor->vm_data.valid) { - exit->temperature = initial_temperature_backoff_counter(); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(exit->executor); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (exit->executor == NULL) { - _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - } - else { - int chain_depth = current_executor->vm_data.chain_depth + 1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } - exit->temperature = initial_temperature_backoff_counter(); - } - exit->executor = executor; - } + tstate->jit_exit = exit; GOTO_TIER_TWO(exit->executor); break; } @@ -7438,7 +7402,19 @@ #ifndef _Py_JIT current_executor = (_PyExecutorObject*)executor; #endif - assert(((_PyExecutorObject *)executor)->vm_data.valid); + assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); + tstate->current_executor = (PyObject *)executor; + if (!current_executor->vm_data.valid) { + assert(tstate->jit_exit->executor == current_executor); + assert(tstate->current_executor == executor); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyExecutor_ClearExit(tstate->jit_exit); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } break; } @@ -7461,6 +7437,14 @@ break; } + case _HANDLE_PENDING_AND_DEOPT: { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_HandlePending(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + break; + } + case _ERROR_POP_N: { oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); @@ -7487,4 +7471,40 @@ break; } + case _COLD_EXIT: { + _PyExitData *exit = tstate->jit_exit; + assert(exit != NULL); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + 1; + _PyFrame_SetStackPointer(frame, stack_pointer); + int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (optimized <= 0) { + exit->temperature = restart_backoff_counter(temperature); + GOTO_TIER_ONE(optimized < 0 ? NULL : target); + } + exit->temperature = initial_temperature_backoff_counter(); + } + assert(tstate->jit_exit == exit); + exit->executor = executor; + GOTO_TIER_TWO(exit->executor); + break; + } + #undef TIER_TWO diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 1cb6f03169e3b5..d82da15a43cac6 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -3472,11 +3472,13 @@ convert_pseudo_conditional_jumps(cfg_builder *g) instr->i_opcode = instr->i_opcode == JUMP_IF_FALSE ? POP_JUMP_IF_FALSE : POP_JUMP_IF_TRUE; location loc = instr->i_loc; + basicblock *except = instr->i_except; cfg_instr copy = { .i_opcode = COPY, .i_oparg = 1, .i_loc = loc, .i_target = NULL, + .i_except = except, }; RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, ©)); cfg_instr to_bool = { @@ -3484,6 +3486,7 @@ convert_pseudo_conditional_jumps(cfg_builder *g) .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = except, }; RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, &to_bool)); } @@ -3726,6 +3729,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &make_gen)); cfg_instr pop_top = { @@ -3733,6 +3737,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 1, &pop_top)); } @@ -3763,6 +3768,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = oldindex, .i_loc = NO_LOCATION, .i_target = NULL, + .i_except = NULL, }; if (basicblock_insert_instruction(entryblock, ncellsused, &make_cell) < 0) { PyMem_RawFree(sorted); @@ -3779,6 +3785,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = nfreevars, .i_loc = NO_LOCATION, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, ©_frees)); } diff --git a/Python/frozenmain.c b/Python/frozenmain.c index ec4566bd4f84bc..3de587c0423226 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -1,8 +1,7 @@ /* Python interpreter main program for frozen scripts */ #include "Python.h" -#include "pycore_pystate.h" // _Py_GetConfig() -#include "pycore_runtime.h" // _PyRuntime_Initialize() +#include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #ifdef HAVE_UNISTD_H # include // isatty() @@ -20,11 +19,6 @@ extern int PyInitFrozenExtensions(void); int Py_FrozenMain(int argc, char **argv) { - PyStatus status = _PyRuntime_Initialize(); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - PyConfig config; PyConfig_InitPythonConfig(&config); // Suppress errors from getpath.c @@ -32,7 +26,7 @@ Py_FrozenMain(int argc, char **argv) // Don't parse command line options like -E config.parse_argv = 0; - status = PyConfig_SetBytesArgv(&config, argc, argv); + PyStatus status = PyConfig_SetBytesArgv(&config, argc, argv); if (PyStatus_Exception(status)) { PyConfig_Clear(&config); Py_ExitStatusException(status); @@ -64,7 +58,12 @@ Py_FrozenMain(int argc, char **argv) PyWinFreeze_ExeInit(); #endif - if (_Py_GetConfig()->verbose) { + int verbose; + if (PyConfig_GetInt("verbose", &verbose) < 0) { + verbose = 0; + PyErr_Clear(); + } + if (verbose) { fprintf(stderr, "Python %s\n%s\n", Py_GetVersion(), Py_GetCopyright()); } diff --git a/Python/gc.c b/Python/gc.c index 2e697faac032b5..03455e88d5eeb1 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -570,8 +570,7 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) } /* Subtract internal references from gc_refs. After this, gc_refs is >= 0 - * for all objects in containers, and is GC_REACHABLE for all tracked gc - * objects not in containers. The ones with gc_refs > 0 are directly + * for all objects in containers. The ones with gc_refs > 0 are directly * reachable from outside containers, and so can't be collected. */ static void @@ -858,60 +857,65 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers) } } -/* Clear all weakrefs to unreachable objects, and if such a weakref has a - * callback, invoke it if necessary. Note that it's possible for such - * weakrefs to be outside the unreachable set -- indeed, those are precisely - * the weakrefs whose callbacks must be invoked. See gc_weakref.txt for - * overview & some details. Some weakrefs with callbacks may be reclaimed - * directly by this routine; the number reclaimed is the return value. Other - * weakrefs with callbacks may be moved into the `old` generation. Objects - * moved into `old` have gc_refs set to GC_REACHABLE; the objects remaining in - * unreachable are left at GC_TENTATIVELY_UNREACHABLE. When this returns, - * no object in `unreachable` is weakly referenced anymore. +/* Handle weakref callbacks. Note that it's possible for such weakrefs to be + * outside the unreachable set -- indeed, those are precisely the weakrefs + * whose callbacks must be invoked. See gc_weakref.txt for overview & some + * details. + * + * The clearing of weakrefs is suble and must be done carefully, as there was + * previous bugs related to this. First, weakrefs to the unreachable set of + * objects must be cleared before we start calling `tp_clear`. If we don't, + * those weakrefs can reveal unreachable objects to Python-level code and that + * is not safe. Many objects are not usable after `tp_clear` has been called + * and could even cause crashes if accessed (see bpo-38006 and gh-91636 as + * example bugs). + * + * Weakrefs with callbacks always need to be cleared before executing the + * callback. That's because sometimes a callback will call the ref object, + * to check if the reference is actually dead (KeyedRef does this, for + * example). We want to indicate that it is dead, even though it is possible + * a finalizer might resurrect it. Clearing also prevents the callback from + * executing more than once. + * + * Since Python 2.3, all weakrefs to cyclic garbage have been cleared *before* + * calling finalizers. However, since tp_subclasses started being necessary + * to invalidate caches (e.g. by PyType_Modified), that clearing has created + * a bug. If the weakref to the subclass is cleared before a finalizer is + * called, the cache may not be correctly invalidated. That can lead to + * segfaults since the caches can refer to deallocated objects (GH-91636 + * is an example). Now, we delay the clear of weakrefs without callbacks + * until *after* finalizers have been executed. That means weakrefs without + * callbacks are still usable while finalizers are being executed. + * + * The weakrefs that are inside the unreachable set must also be cleared. + * The reason this is required is not immediately obvious. If the weakref + * refers to an object outside of the unreachable set, that object might go + * away when we start clearing objects. Normally, the object should also be + * part of the unreachable set but that's not true in the case of incomplete + * or missing `tp_traverse` methods. When that object goes away, the callback + * for weakref can be executed and that could reveal unreachable objects to + * Python-level code. See bpo-38006 as an example bug. */ static int -handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) +handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old) { PyGC_Head *gc; - PyObject *op; /* generally FROM_GC(gc) */ - PyWeakReference *wr; /* generally a cast of op */ PyGC_Head wrcb_to_call; /* weakrefs with callbacks to call */ PyGC_Head *next; int num_freed = 0; - if (allow_callbacks) { - gc_list_init(&wrcb_to_call); - } + gc_list_init(&wrcb_to_call); - /* Clear all weakrefs to the objects in unreachable. If such a weakref - * also has a callback, move it into `wrcb_to_call` if the callback - * needs to be invoked. Note that we cannot invoke any callbacks until - * all weakrefs to unreachable objects are cleared, lest the callback - * resurrect an unreachable object via a still-active weakref. We - * make another pass over wrcb_to_call, invoking callbacks, after this - * pass completes. + /* Find all weakrefs with callbacks and move into `wrcb_to_call` if the + * callback needs to be invoked. We make another pass over wrcb_to_call, + * invoking callbacks, after this pass completes. */ for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { PyWeakReference **wrlist; - op = FROM_GC(gc); + PyObject *op = FROM_GC(gc); next = GC_NEXT(gc); - if (PyWeakref_Check(op)) { - /* A weakref inside the unreachable set must be cleared. If we - * allow its callback to execute inside delete_garbage(), it - * could expose objects that have tp_clear already called on - * them. Or, it could resurrect unreachable objects. One way - * this can happen is if some container objects do not implement - * tp_traverse. Then, wr_object can be outside the unreachable - * set but can be deallocated as a result of breaking the - * reference cycle. If we don't clear the weakref, the callback - * will run and potentially cause a crash. See bpo-38006 for - * one example. - */ - _PyWeakref_ClearRef((PyWeakReference *)op); - } - if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { continue; } @@ -923,30 +927,29 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) */ wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); - /* `op` may have some weakrefs. March over the list, clear - * all the weakrefs, and move the weakrefs with callbacks - * that must be called into wrcb_to_call. + /* `op` may have some weakrefs. March over the list and move the + * weakrefs with callbacks that must be called into wrcb_to_call. */ - for (wr = *wrlist; wr != NULL; wr = *wrlist) { - PyGC_Head *wrasgc; /* AS_GC(wr) */ - - /* _PyWeakref_ClearRef clears the weakref but leaves - * the callback pointer intact. Obscure: it also - * changes *wrlist. - */ - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); - _PyWeakref_ClearRef(wr); - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); - - if (!allow_callbacks) { - continue; - } + PyWeakReference *next_wr; + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = next_wr) { + // Get the next list element to get iterator progress if we omit + // clearing of the weakref (because _PyWeakref_ClearRef changes + // next pointer in the wrlist). + next_wr = wr->wr_next; if (wr->wr_callback == NULL) { /* no callback */ continue; } + // Clear the weakref. See the comments above this function for + // reasons why we need to clear weakrefs that have callbacks. + // Note that _PyWeakref_ClearRef clears the weakref but leaves the + // callback pointer intact. Obscure: it also changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + /* Headache time. `op` is going away, and is weakly referenced by * `wr`, which has a callback. Should the callback be invoked? If wr * is also trash, no: @@ -962,10 +965,10 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) * outside the current generation, CT may be reachable from the * callback. Then the callback could resurrect insane objects. * - * Since the callback is never needed and may be unsafe in this case, - * wr is simply left in the unreachable set. Note that because we - * already called _PyWeakref_ClearRef(wr), its callback will never - * trigger. + * Since the callback is never needed and may be unsafe in this + * case, wr is simply left in the unreachable set. Note that + * clear_weakrefs() will ensure its callback will not trigger + * inside delete_garbage(). * * OTOH, if wr isn't part of CT, we should invoke the callback: the * weakref outlived the trash. Note that since wr isn't CT in this @@ -976,8 +979,6 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) * is moved to wrcb_to_call in this case. */ if (gc_is_collecting(AS_GC((PyObject *)wr))) { - /* it should already have been cleared above */ - _PyObject_ASSERT((PyObject*)wr, wr->wr_object == Py_None); continue; } @@ -987,17 +988,13 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) Py_INCREF(wr); /* Move wr to wrcb_to_call, for the next pass. */ - wrasgc = AS_GC((PyObject *)wr); + PyGC_Head *wrasgc = AS_GC((PyObject *)wr); // wrasgc is reachable, but next isn't, so they can't be the same _PyObject_ASSERT((PyObject *)wr, wrasgc != next); gc_list_move(wrasgc, &wrcb_to_call); } } - if (!allow_callbacks) { - return 0; - } - /* Invoke the callbacks we decided to honor. It's safe to invoke them * because they can't reference unreachable objects. */ @@ -1007,9 +1004,9 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) PyObject *callback; gc = (PyGC_Head*)wrcb_to_call._gc_next; - op = FROM_GC(gc); + PyObject *op = FROM_GC(gc); _PyObject_ASSERT(op, PyWeakref_Check(op)); - wr = (PyWeakReference *)op; + PyWeakReference *wr = (PyWeakReference *)op; callback = wr->wr_callback; _PyObject_ASSERT(op, callback != NULL); @@ -1048,6 +1045,56 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) return num_freed; } +/* Clear all weakrefs to unreachable objects. When this returns, no object in + * `unreachable` is weakly referenced anymore. See the comments above + * handle_weakref_callbacks() for why these weakrefs need to be cleared. + */ +static void +clear_weakrefs(PyGC_Head *unreachable) +{ + PyGC_Head *gc; + PyGC_Head *next; + + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { + PyWeakReference **wrlist; + + PyObject *op = FROM_GC(gc); + next = GC_NEXT(gc); + + if (PyWeakref_Check(op)) { + /* A weakref inside the unreachable set is always cleared. See + * the comments above handle_weakref_callbacks() for why these + * must be cleared. + */ + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + /* It supports weakrefs. Does it have any? + * + * This is never triggered for static types so we can avoid the + * (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + */ + wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + /* `op` may have some weakrefs. March over the list, clear + * all the weakrefs. + */ + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { + /* _PyWeakref_ClearRef clears the weakref but leaves + * the callback pointer intact. Obscure: it also + * changes *wrlist. + */ + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + } + } +} + static void debug_cycle(const char *msg, PyObject *op) { @@ -1710,12 +1757,13 @@ gc_collect_region(PyThreadState *tstate, deduce_unreachable(from, &unreachable); validate_consistent_old_space(from); untrack_tuples(from); + + /* Move reachable objects to next generation. */ validate_consistent_old_space(to); if (from != to) { gc_list_merge(from, to); } validate_consistent_old_space(to); - /* Move reachable objects to next generation. */ /* All objects in unreachable are trash, but objects reachable from * legacy finalizers (e.g. tp_del) can't safely be deleted. @@ -1738,8 +1786,8 @@ gc_collect_region(PyThreadState *tstate, } } - /* Clear weakrefs and invoke callbacks as necessary. */ - stats->collected += handle_weakrefs(&unreachable, to, true); + /* Invoke weakref callbacks as necessary. */ + stats->collected += handle_weakref_callbacks(&unreachable, to); gc_list_validate_space(to, gcstate->visited_space); validate_list(to, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); @@ -1753,13 +1801,10 @@ gc_collect_region(PyThreadState *tstate, gc_list_init(&final_unreachable); handle_resurrected_objects(&unreachable, &final_unreachable, to); - /* Clear weakrefs to objects in the unreachable set. No Python-level - * code must be allowed to access those unreachable objects. During - * delete_garbage(), finalizers outside the unreachable set might run - * and create new weakrefs. If those weakrefs were not cleared, they - * could reveal unreachable objects. Callbacks are not executed. + /* Clear weakrefs to objects in the unreachable set. See the comments + * above handle_weakref_callbacks() for details. */ - handle_weakrefs(&final_unreachable, NULL, false); + clear_weakrefs(&final_unreachable); /* Call tp_clear on objects in the final_unreachable set. This will cause * the reference cycles to be broken. It may also cause some objects diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 0b0ddf227e4952..842aa3401548c9 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1492,22 +1492,49 @@ move_legacy_finalizer_reachable(struct collection_state *state) return 0; } -// Clear all weakrefs to unreachable objects. Weakrefs with callbacks are -// optionally enqueued in `wrcb_to_call`, but not invoked yet. +// Weakrefs with callbacks are enqueued in `wrcb_to_call`, but not invoked +// yet. Note that it's possible for such weakrefs to be outside the +// unreachable set -- indeed, those are precisely the weakrefs whose callbacks +// must be invoked. See gc_weakref.txt for overview & some details. +// +// The clearing of weakrefs is suble and must be done carefully, as there was +// previous bugs related to this. First, weakrefs to the unreachable set of +// objects must be cleared before we start calling `tp_clear`. If we don't, +// those weakrefs can reveal unreachable objects to Python-level code and that +// is not safe. Many objects are not usable after `tp_clear` has been called +// and could even cause crashes if accessed (see bpo-38006 and gh-91636 as +// example bugs). +// +// Weakrefs with callbacks always need to be cleared before executing the +// callback. That's because sometimes a callback will call the ref object, +// to check if the reference is actually dead (KeyedRef does this, for +// example). We want to indicate that it is dead, even though it is possible +// a finalizer might resurrect it. Clearing also prevents the callback from +// executing more than once. +// +// Since Python 2.3, all weakrefs to cyclic garbage have been cleared *before* +// calling finalizers. However, since tp_subclasses started being necessary +// to invalidate caches (e.g. by PyType_Modified), that clearing has created +// a bug. If the weakref to the subclass is cleared before a finalizer is +// called, the cache may not be correctly invalidated. That can lead to +// segfaults since the caches can refer to deallocated objects (GH-91636 +// is an example). Now, we delay the clear of weakrefs without callbacks +// until *after* finalizers have been executed. That means weakrefs without +// callbacks are still usable while finalizers are being executed. +// +// The weakrefs that are inside the unreachable set must also be cleared. +// The reason this is required is not immediately obvious. If the weakref +// refers to an object outside of the unreachable set, that object might go +// away when we start clearing objects. Normally, the object should also be +// part of the unreachable set but that's not true in the case of incomplete +// or missing `tp_traverse` methods. When that object goes away, the callback +// for weakref can be executed and that could reveal unreachable objects to +// Python-level code. See bpo-38006 as an example bug. static void -clear_weakrefs(struct collection_state *state, bool enqueue_callbacks) +find_weakref_callbacks(struct collection_state *state) { PyObject *op; WORKSTACK_FOR_EACH(&state->unreachable, op) { - if (PyWeakref_Check(op)) { - // Clear weakrefs that are themselves unreachable to ensure their - // callbacks will not be executed later from a `tp_clear()` - // inside delete_garbage(). That would be unsafe: it could - // resurrect a dead object or access a an already cleared object. - // See bpo-38006 for one example. - _PyWeakref_ClearRef((PyWeakReference *)op); - } - if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { continue; } @@ -1519,16 +1546,21 @@ clear_weakrefs(struct collection_state *state, bool enqueue_callbacks) // `op` may have some weakrefs. March over the list, clear // all the weakrefs, and enqueue the weakrefs with callbacks // that must be called into wrcb_to_call. - for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { - // _PyWeakref_ClearRef clears the weakref but leaves - // the callback pointer intact. Obscure: it also - // changes *wrlist. - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); - _PyWeakref_ClearRef(wr); - _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); - - if (!enqueue_callbacks) { - continue; + PyWeakReference *next_wr; + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = next_wr) { + // Get the next list element to get iterator progress if we omit + // clearing of the weakref (because _PyWeakref_ClearRef changes + // next pointer in the wrlist). + next_wr = wr->wr_next; + + // Weakrefs with callbacks always need to be cleared before + // executing the callback. + if (wr->wr_callback != NULL) { + // _PyWeakref_ClearRef clears the weakref but leaves the + // callback pointer intact. Obscure: it also changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); } // We do not invoke callbacks for weakrefs that are themselves @@ -1550,6 +1582,39 @@ clear_weakrefs(struct collection_state *state, bool enqueue_callbacks) } } +// Clear weakrefs to objects in the unreachable set. See comments +// above find_weakref_callbacks() for why this clearing is required. +static void +clear_weakrefs(struct collection_state *state) +{ + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + if (PyWeakref_Check(op)) { + // Clear weakrefs that are themselves unreachable. + _PyWeakref_ClearRef((PyWeakReference *)op); + } + + if (!_PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + continue; + } + + // NOTE: This is never triggered for static types so we can avoid the + // (slightly) more costly _PyObject_GET_WEAKREFS_LISTPTR(). + PyWeakReference **wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op); + + // `op` may have some weakrefs. March over the list, clear + // all the weakrefs. + for (PyWeakReference *wr = *wrlist; wr != NULL; wr = *wrlist) { + // _PyWeakref_ClearRef clears the weakref but leaves + // the callback pointer intact. Obscure: it also + // changes *wrlist. + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); + _PyWeakref_ClearRef(wr); + _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + } + } +} + static void call_weakref_callbacks(struct collection_state *state) { @@ -2222,8 +2287,8 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Record the number of live GC objects interp->gc.long_lived_total = state->long_lived_total; - // Clear weakrefs and enqueue callbacks (but do not call them). - clear_weakrefs(state, true); + // Find weakref callbacks we will honor (but do not call them). + find_weakref_callbacks(state); _PyEval_StartTheWorld(interp); // Deallocate any object from the refcount merge step @@ -2240,12 +2305,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Clear free lists in all threads _PyGC_ClearAllFreeLists(interp); if (err == 0) { - // Clear weakrefs to objects in the unreachable set. No Python-level - // code must be allowed to access those unreachable objects. During - // delete_garbage(), finalizers outside the unreachable set might - // run and create new weakrefs. If those weakrefs were not cleared, - // they could reveal unreachable objects. - clear_weakrefs(state, false); + clear_weakrefs(state); } _PyEval_StartTheWorld(interp); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index aa1eb373b7ba4b..d683582761295a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1670,26 +1670,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2175,26 +2167,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2292,26 +2276,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2409,26 +2385,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2501,26 +2469,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -2676,26 +2636,18 @@ } result = PyStackRef_FromPyObjectSteal(result_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3268,26 +3220,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3659,26 +3603,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3790,26 +3726,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -3891,26 +3819,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4005,26 +3925,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4128,26 +4040,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4421,26 +4325,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -4501,26 +4397,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -5595,6 +5483,8 @@ } DISPATCH_GOTO(); } + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; GOTO_TIER_TWO(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -6515,26 +6405,18 @@ } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -6690,26 +6572,18 @@ } result = PyStackRef_FromPyObjectSteal(result_o); } - // _CHECK_PERIODIC + // _CHECK_PERIODIC_AT_END { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } - stack_pointer += -1; + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -7091,15 +6965,11 @@ /* Skip 1 cache entry */ // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _MONITOR_JUMP_BACKWARD @@ -7482,15 +7352,11 @@ // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } } @@ -7719,15 +7585,11 @@ } // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7752,15 +7614,11 @@ /* Skip 1 cache entry */ // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7793,6 +7651,8 @@ this_instr[1].counter = initial_jump_backoff_counter(); stack_pointer = _PyFrame_GetStackPointer(frame); assert(tstate->current_executor == NULL); + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; GOTO_TIER_TWO(executor); } } @@ -7829,15 +7689,11 @@ /* Skip 1 cache entry */ // _CHECK_PERIODIC { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -10415,15 +10271,11 @@ // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - JUMP_TO_LABEL(error); - } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); } } } diff --git a/Python/hamt.c b/Python/hamt.c index 906149cc6cdbdc..e372b1a1b4c18b 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -256,9 +256,9 @@ Debug ===== The HAMT datatype is accessible for testing purposes under the -`_testcapi` module: +`_testinternalcapi` module: - >>> from _testcapi import hamt + >>> from _testinternalcapi import hamt >>> h = hamt() >>> h2 = h.set('a', 2) >>> h3 = h2.set('b', 3) diff --git a/Python/jit.c b/Python/jit.c index 01bc0076497c6d..bd6a5e17a4164f 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -69,10 +69,6 @@ jit_alloc(size_t size) #else int flags = MAP_ANONYMOUS | MAP_PRIVATE; int prot = PROT_READ | PROT_WRITE; -# ifdef MAP_JIT - flags |= MAP_JIT; - prot |= PROT_EXEC; -# endif unsigned char *memory = mmap(NULL, size, prot, flags, -1, 0); int failed = memory == MAP_FAILED; #endif @@ -118,11 +114,8 @@ mark_executable(unsigned char *memory, size_t size) int old; int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old); #else - int failed = 0; __builtin___clear_cache((char *)memory, (char *)memory + size); -#ifndef MAP_JIT - failed = mprotect(memory, size, PROT_EXEC | PROT_READ); -#endif + int failed = mprotect(memory, size, PROT_EXEC | PROT_READ); #endif if (failed) { jit_error("unable to protect executable memory"); @@ -531,9 +524,6 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz if (memory == NULL) { return -1; } -#ifdef MAP_JIT - pthread_jit_write_protect_np(0); -#endif // Collect memory stats OPT_STAT_ADD(jit_total_memory_size, total_size); OPT_STAT_ADD(jit_code_size, code_size); @@ -556,7 +546,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz group->emit(code, data, executor, NULL, &state); code += group->code_size; data += group->data_size; - assert(trace[0].opcode == _START_EXECUTOR); + assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; @@ -571,9 +561,6 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz data += group->data_size; assert(code == memory + code_size); assert(data == memory + code_size + state.trampolines.size + code_padding + data_size); -#ifdef MAP_JIT - pthread_jit_write_protect_np(1); -#endif if (mark_executable(memory, total_size)) { jit_free(memory, total_size); return -1; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index dbd19d7755c237..ee9b0cd0e9cfc3 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -20,13 +20,6 @@ typedef struct _PyLegacyEventHandler { #define _PyLegacyEventHandler_CAST(op) ((_PyLegacyEventHandler *)(op)) -#ifdef Py_GIL_DISABLED -#define LOCK_SETUP() PyMutex_Lock(&_PyRuntime.ceval.sys_trace_profile_mutex); -#define UNLOCK_SETUP() PyMutex_Unlock(&_PyRuntime.ceval.sys_trace_profile_mutex); -#else -#define LOCK_SETUP() -#define UNLOCK_SETUP() -#endif /* The Py_tracefunc function expects the following arguments: * obj: the trace object (PyObject *) * frame: the current frame (PyFrameObject *) @@ -509,9 +502,9 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) // needs to be decref'd outside of the lock PyObject *old_profileobj; - LOCK_SETUP(); + FT_MUTEX_LOCK(&_PyRuntime.ceval.sys_trace_profile_mutex); Py_ssize_t profiling_threads = setup_profile(tstate, func, arg, &old_profileobj); - UNLOCK_SETUP(); + FT_MUTEX_UNLOCK(&_PyRuntime.ceval.sys_trace_profile_mutex); Py_XDECREF(old_profileobj); uint32_t events = 0; @@ -605,10 +598,10 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) } // needs to be decref'd outside of the lock PyObject *old_traceobj; - LOCK_SETUP(); + FT_MUTEX_LOCK(&_PyRuntime.ceval.sys_trace_profile_mutex); assert(tstate->interp->sys_tracing_threads >= 0); Py_ssize_t tracing_threads = setup_tracing(tstate, func, arg, &old_traceobj); - UNLOCK_SETUP(); + FT_MUTEX_UNLOCK(&_PyRuntime.ceval.sys_trace_profile_mutex); Py_XDECREF(old_traceobj); if (tracing_threads < 0) { return -1; diff --git a/Python/optimizer.c b/Python/optimizer.c index 8d01d605ef4a2a..f06df9644a1614 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -205,8 +205,8 @@ static int executor_clear(PyObject *executor); static void unlink_executor(_PyExecutorObject *executor); -static void -free_executor(_PyExecutorObject *self) +void +_PyExecutor_Free(_PyExecutorObject *self) { #ifdef _Py_JIT _PyJIT_Free(self); @@ -242,7 +242,7 @@ _Py_ClearExecutorDeletionList(PyInterpreterState *interp) } else { *prev_to_next_ptr = exec->vm_data.links.next; - free_executor(exec); + _PyExecutor_Free(exec); } exec = *prev_to_next_ptr; } @@ -432,6 +432,7 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, [_FOR_ITER] = _FOR_ITER_TIER_TWO, [_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO, + [_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK, }; static const uint8_t @@ -776,9 +777,12 @@ translate_bytecode_to_trace( case OPARG_REPLACED: uop = _PyUOp_Replacements[uop]; assert(uop != 0); + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); + if (uop == _TIER2_RESUME_CHECK) { + target = next_inst; + } #ifdef Py_DEBUG - { - uint32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + (oparg > 255); + else { uint32_t jump_target = next_inst + oparg; assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); @@ -1043,9 +1047,15 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) _PyUOpInstruction *inst = &buffer[i]; int opcode = inst->opcode; int32_t target = (int32_t)uop_get_target(inst); - if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { - uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? - _EXIT_TRACE : _DEOPT; + uint16_t exit_flags = _PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG | HAS_PERIODIC_FLAG); + if (exit_flags) { + uint16_t exit_op = _EXIT_TRACE; + if (exit_flags & HAS_DEOPT_FLAG) { + exit_op = _DEOPT; + } + else if (exit_flags & HAS_PERIODIC_FLAG) { + exit_op = _HANDLE_PENDING_AND_DEOPT; + } int32_t jump_target = target; if (is_for_iter_test[opcode]) { /* Target the POP_TOP immediately after the END_FOR, @@ -1129,7 +1139,7 @@ sanity_check(_PyExecutorObject *executor) } bool ended = false; uint32_t i = 0; - CHECK(executor->trace[0].opcode == _START_EXECUTOR); + CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; @@ -1159,6 +1169,7 @@ sanity_check(_PyExecutorObject *executor) uint16_t opcode = inst->opcode; CHECK( opcode == _DEOPT || + opcode == _HANDLE_PENDING_AND_DEOPT || opcode == _EXIT_TRACE || opcode == _ERROR_POP_N); } @@ -1182,9 +1193,11 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } /* Initialize exits */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); for (int i = 0; i < exit_count; i++) { - executor->exits[i].executor = NULL; + executor->exits[i].index = i; executor->exits[i].temperature = initial_temperature_backoff_counter(); + executor->exits[i].executor = cold; } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; @@ -1462,6 +1475,46 @@ _Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_s link_executor(executor); } +_PyExecutorObject * +_PyExecutor_GetColdExecutor(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->cold_executor != NULL) { + return interp->cold_executor; + } + _PyExecutorObject *cold = allocate_executor(0, 1); + if (cold == NULL) { + Py_FatalError("Cannot allocate core JIT code"); + } + ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_EXIT; +#ifdef _Py_JIT + cold->jit_code = NULL; + cold->jit_side_entry = NULL; + cold->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + cold->vm_data.warm = true; + if (_PyJIT_Compile(cold, cold->trace, 1)) { + Py_DECREF(cold); + Py_FatalError("Cannot allocate core JIT code"); + } +#endif + _Py_SetImmortal((PyObject *)cold); + interp->cold_executor = cold; + return cold; +} + +void +_PyExecutor_ClearExit(_PyExitData *exit) +{ + if (exit == NULL) { + return; + } + _PyExecutorObject *old = exit->executor; + exit->executor = _PyExecutor_GetColdExecutor(); + Py_DECREF(old); +} + /* Detaches the executor from the code object (if any) that * holds a reference to it */ void @@ -1492,14 +1545,18 @@ executor_clear(PyObject *op) assert(executor->vm_data.valid == 1); unlink_executor(executor); executor->vm_data.valid = 0; + /* It is possible for an executor to form a reference * cycle with itself, so decref'ing a side exit could * free the executor unless we hold a strong reference to it */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); Py_INCREF(executor); for (uint32_t i = 0; i < executor->exit_count; i++) { executor->exits[i].temperature = initial_unreachable_backoff_counter(); - Py_CLEAR(executor->exits[i].executor); + _PyExecutorObject *e = executor->exits[i].executor; + executor->exits[i].executor = cold; + Py_DECREF(e); } _Py_ExecutorDetach(executor); Py_DECREF(executor); @@ -1741,4 +1798,11 @@ _PyDumpExecutors(FILE *out) return -1; } +void +_PyExecutor_Free(struct _PyExecutorObject *self) +{ + /* This should never be called */ + Py_UNREACHABLE(); +} + #endif /* _Py_TIER2 */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 99206f01618d79..2477ede3e68017 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -11,6 +11,8 @@ break; } + /* _CHECK_PERIODIC_AT_END is not a viable micro-op for tier 2 */ + case _CHECK_PERIODIC_IF_NOT_YIELD_FROM: { break; } @@ -3198,6 +3200,10 @@ break; } + case _HANDLE_PENDING_AND_DEOPT: { + break; + } + case _ERROR_POP_N: { break; } @@ -3206,3 +3212,7 @@ break; } + case _COLD_EXIT: { + break; + } + diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index a2da3c7d56df50..987e8d2a11a659 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -202,6 +202,7 @@ enum perf_trampoline_type { #define perf_map_file _PyRuntime.ceval.perf.map_file #define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork #define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type +#define prev_eval_frame _PyRuntime.ceval.perf.prev_eval_frame static void perf_map_write_entry(void *state, const void *code_addr, @@ -407,9 +408,12 @@ py_trampoline_evaluator(PyThreadState *ts, _PyInterpreterFrame *frame, f = new_trampoline; } assert(f != NULL); - return f(ts, frame, throw, _PyEval_EvalFrameDefault); + return f(ts, frame, throw, prev_eval_frame != NULL ? prev_eval_frame : _PyEval_EvalFrameDefault); default_eval: // Something failed, fall back to the default evaluator. + if (prev_eval_frame) { + return prev_eval_frame(ts, frame, throw); + } return _PyEval_EvalFrameDefault(ts, frame, throw); } #endif // PY_HAVE_PERF_TRAMPOLINE @@ -481,18 +485,12 @@ _PyPerfTrampoline_Init(int activate) { #ifdef PY_HAVE_PERF_TRAMPOLINE PyThreadState *tstate = _PyThreadState_GET(); - if (tstate->interp->eval_frame && - tstate->interp->eval_frame != py_trampoline_evaluator) { - PyErr_SetString(PyExc_RuntimeError, - "Trampoline cannot be initialized as a custom eval " - "frame is already present"); - return -1; - } if (!activate) { - _PyInterpreterState_SetEvalFrameFunc(tstate->interp, NULL); + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, prev_eval_frame); perf_status = PERF_STATUS_NO_INIT; } - else { + else if (tstate->interp->eval_frame != py_trampoline_evaluator) { + prev_eval_frame = _PyInterpreterState_GetEvalFrameFunc(tstate->interp); _PyInterpreterState_SetEvalFrameFunc(tstate->interp, py_trampoline_evaluator); extra_code_index = _PyEval_RequestCodeExtraIndex(NULL); if (extra_code_index == -1) { diff --git a/Python/pystate.c b/Python/pystate.c index 04ca6edb4aaa0e..cd62bf86837f83 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -815,7 +815,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) /* Last garbage collection on this interpreter */ _PyGC_CollectNoFail(tstate); _PyGC_Fini(interp); - + struct _PyExecutorObject *cold = interp->cold_executor; + if (cold != NULL) { + interp->cold_executor = NULL; + assert(cold->vm_data.valid); + assert(cold->vm_data.warm); + _PyExecutor_Free(cold); + } /* We don't clear sysdict and builtins until the end of this function. Because clearing other attributes can execute arbitrary Python code which requires sysdict and builtins. */ @@ -1469,6 +1475,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_limit = NULL; tstate->what_event = -1; tstate->current_executor = NULL; + tstate->jit_exit = NULL; tstate->dict_global_version = 0; _tstate->c_stack_soft_limit = UINTPTR_MAX; @@ -1682,9 +1689,7 @@ PyThreadState_Clear(PyThreadState *tstate) "PyThreadState_Clear: warning: thread still has a generator\n"); } -#ifdef Py_GIL_DISABLED - PyMutex_Lock(&_PyRuntime.ceval.sys_trace_profile_mutex); -#endif + FT_MUTEX_LOCK(&_PyRuntime.ceval.sys_trace_profile_mutex); if (tstate->c_profilefunc != NULL) { tstate->interp->sys_profiling_threads--; @@ -1695,9 +1700,7 @@ PyThreadState_Clear(PyThreadState *tstate) tstate->c_tracefunc = NULL; } -#ifdef Py_GIL_DISABLED - PyMutex_Unlock(&_PyRuntime.ceval.sys_trace_profile_mutex); -#endif + FT_MUTEX_UNLOCK(&_PyRuntime.ceval.sys_trace_profile_mutex); Py_CLEAR(tstate->c_profileobj); Py_CLEAR(tstate->c_traceobj); diff --git a/Python/specialize.c b/Python/specialize.c index fe8d04cf3442f1..38df5741f32520 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -935,8 +935,7 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, un PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)); has_getattr = getattr != NULL; if (has_custom_getattribute) { - if (getattro_slot == _Py_slot_tp_getattro && - !has_getattr && + if (!has_getattr && Py_IS_TYPE(getattribute, &PyFunction_Type)) { *descr = getattribute; *tp_version = ga_version; @@ -1259,12 +1258,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* return -1; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { - #ifndef Py_GIL_DISABLED - // In free-threaded builds it's possible for tp_getattro to change - // after the call to analyze_descriptor. That is fine: the version - // guard will fail. - assert(type->tp_getattro == _Py_slot_tp_getattro); - #endif assert(Py_IS_TYPE(descr, &PyFunction_Type)); _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1); if (!function_check_args(descr, 2, LOAD_ATTR)) { diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 6466d2615cd14e..9dd7e5dbfbae7b 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -5,7 +5,7 @@ import re from typing import Optional, Callable -from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, WhileStmt +from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, WhileStmt, ForStmt, MacroIfStmt @dataclass class EscapingCall: @@ -20,6 +20,7 @@ class Properties: error_with_pop: bool error_without_pop: bool deopts: bool + deopts_periodic: bool oparg: bool jumps: bool eval_breaker: bool @@ -58,6 +59,7 @@ def from_list(properties: list["Properties"]) -> "Properties": error_with_pop=any(p.error_with_pop for p in properties), error_without_pop=any(p.error_without_pop for p in properties), deopts=any(p.deopts for p in properties), + deopts_periodic=any(p.deopts_periodic for p in properties), oparg=any(p.oparg for p in properties), jumps=any(p.jumps for p in properties), eval_breaker=any(p.eval_breaker for p in properties), @@ -85,6 +87,7 @@ def infallible(self) -> bool: error_with_pop=False, error_without_pop=False, deopts=False, + deopts_periodic=False, oparg=False, jumps=False, eval_breaker=False, @@ -706,7 +709,7 @@ def visit(stmt: Stmt) -> None: in_if = 0 tkn_iter = iter(stmt.contents) for tkn in tkn_iter: - if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF", "EXIT_IF"): + if tkn.kind == "IDENTIFIER" and tkn.text in ("DEOPT_IF", "ERROR_IF", "EXIT_IF", "HANDLE_PENDING_AND_DEOPT_IF", "AT_END_EXIT_IF"): in_if = 1 next(tkn_iter) elif tkn.kind == "LPAREN": @@ -723,53 +726,57 @@ def visit(stmt: Stmt) -> None: if error is not None: raise analysis_error(f"Escaping call '{error.text} in condition", error) +def escaping_call_in_simple_stmt(stmt: SimpleStmt, result: dict[SimpleStmt, EscapingCall]) -> None: + tokens = stmt.contents + for idx, tkn in enumerate(tokens): + try: + next_tkn = tokens[idx+1] + except IndexError: + break + if next_tkn.kind != lexer.LPAREN: + continue + if tkn.kind == lexer.IDENTIFIER: + if tkn.text.upper() == tkn.text: + # simple macro + continue + #if not tkn.text.startswith(("Py", "_Py", "monitor")): + # continue + if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): + # Optimize functions + continue + if tkn.text.endswith("Check"): + continue + if tkn.text.startswith("Py_Is"): + continue + if tkn.text.endswith("CheckExact"): + continue + if tkn.text in NON_ESCAPING_FUNCTIONS: + continue + elif tkn.kind == "RPAREN": + prev = tokens[idx-1] + if prev.text.endswith("_t") or prev.text == "*" or prev.text == "int": + #cast + continue + elif tkn.kind != "RBRACKET": + continue + if tkn.text in ("PyStackRef_CLOSE", "PyStackRef_XCLOSE"): + if len(tokens) <= idx+2: + raise analysis_error("Unexpected end of file", next_tkn) + kills = tokens[idx+2] + if kills.kind != "IDENTIFIER": + raise analysis_error(f"Expected identifier, got '{kills.text}'", kills) + else: + kills = None + result[stmt] = EscapingCall(stmt, tkn, kills) + + def find_escaping_api_calls(instr: parser.CodeDef) -> dict[SimpleStmt, EscapingCall]: result: dict[SimpleStmt, EscapingCall] = {} def visit(stmt: Stmt) -> None: if not isinstance(stmt, SimpleStmt): return - tokens = stmt.contents - for idx, tkn in enumerate(tokens): - try: - next_tkn = tokens[idx+1] - except IndexError: - break - if next_tkn.kind != lexer.LPAREN: - continue - if tkn.kind == lexer.IDENTIFIER: - if tkn.text.upper() == tkn.text: - # simple macro - continue - #if not tkn.text.startswith(("Py", "_Py", "monitor")): - # continue - if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): - # Optimize functions - continue - if tkn.text.endswith("Check"): - continue - if tkn.text.startswith("Py_Is"): - continue - if tkn.text.endswith("CheckExact"): - continue - if tkn.text in NON_ESCAPING_FUNCTIONS: - continue - elif tkn.kind == "RPAREN": - prev = tokens[idx-1] - if prev.text.endswith("_t") or prev.text == "*" or prev.text == "int": - #cast - continue - elif tkn.kind != "RBRACKET": - continue - if tkn.text in ("PyStackRef_CLOSE", "PyStackRef_XCLOSE"): - if len(tokens) <= idx+2: - raise analysis_error("Unexpected end of file", next_tkn) - kills = tokens[idx+2] - if kills.kind != "IDENTIFIER": - raise analysis_error(f"Expected identifier, got '{kills.text}'", kills) - else: - kills = None - result[stmt] = EscapingCall(stmt, tkn, kills) + escaping_call_in_simple_stmt(stmt, result) instr.block.accept(visit) check_escaping_calls(instr, result) @@ -822,6 +829,60 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool: ) +def stmt_is_simple_exit(stmt: Stmt) -> bool: + if not isinstance(stmt, SimpleStmt): + return False + tokens = stmt.contents + if len(tokens) < 4: + return False + return ( + tokens[0].text in ("ERROR_IF", "DEOPT_IF", "EXIT_IF", "AT_END_EXIT_IF") + and + tokens[1].text == "(" + and + tokens[2].text in ("true", "1") + and + tokens[3].text == ")" + ) + + +def stmt_list_escapes(stmts: list[Stmt]) -> bool: + if not stmts: + return False + if stmt_is_simple_exit(stmts[-1]): + return False + for stmt in stmts: + if stmt_escapes(stmt): + return True + return False + + +def stmt_escapes(stmt: Stmt) -> bool: + if isinstance(stmt, BlockStmt): + return stmt_list_escapes(stmt.body) + elif isinstance(stmt, SimpleStmt): + for tkn in stmt.contents: + if tkn.text == "DECREF_INPUTS": + return True + d: dict[SimpleStmt, EscapingCall] = {} + escaping_call_in_simple_stmt(stmt, d) + return bool(d) + elif isinstance(stmt, IfStmt): + if stmt.else_body and stmt_escapes(stmt.else_body): + return True + return stmt_escapes(stmt.body) + elif isinstance(stmt, MacroIfStmt): + if stmt.else_body and stmt_list_escapes(stmt.else_body): + return True + return stmt_list_escapes(stmt.body) + elif isinstance(stmt, ForStmt): + return stmt_escapes(stmt.body) + elif isinstance(stmt, WhileStmt): + return stmt_escapes(stmt.body) + else: + assert False, "Unexpected statement type" + + def compute_properties(op: parser.CodeDef) -> Properties: escaping_calls = find_escaping_api_calls(op) has_free = ( @@ -831,11 +892,13 @@ def compute_properties(op: parser.CodeDef) -> Properties: or variable_used(op, "PyCell_SwapTakeRef") ) deopts_if = variable_used(op, "DEOPT_IF") - exits_if = variable_used(op, "EXIT_IF") - if deopts_if and exits_if: + exits_if = variable_used(op, "EXIT_IF") or variable_used(op, "AT_END_EXIT_IF") + deopts_periodic = variable_used(op, "HANDLE_PENDING_AND_DEOPT_IF") + exits_and_deopts = sum((deopts_if, exits_if, deopts_periodic)) + if exits_and_deopts > 1: tkn = op.tokens[0] raise lexer.make_syntax_error( - "Op cannot contain both EXIT_IF and DEOPT_IF", + "Op cannot contain more than one of EXIT_IF, DEOPT_IF and HANDLE_PENDING_AND_DEOPT_IF", tkn.filename, tkn.line, tkn.column, @@ -843,7 +906,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: ) error_with_pop = has_error_with_pop(op) error_without_pop = has_error_without_pop(op) - escapes = bool(escaping_calls) or variable_used(op, "DECREF_INPUTS") + escapes = stmt_escapes(op.block) pure = False if isinstance(op, parser.LabelDef) else "pure" in op.annotations no_save_ip = False if isinstance(op, parser.LabelDef) else "no_save_ip" in op.annotations return Properties( @@ -852,6 +915,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: error_with_pop=error_with_pop, error_without_pop=error_without_pop, deopts=deopts_if, + deopts_periodic=deopts_periodic, side_exit=exits_if, oparg=oparg_used(op), jumps=variable_used(op, "JUMPBY"), diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 4c210fbf8d28e9..cc0edcdf6006fd 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -111,7 +111,9 @@ class Emitter: def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): self._replacers = { "EXIT_IF": self.exit_if, + "AT_END_EXIT_IF": self.exit_if_after, "DEOPT_IF": self.deopt_if, + "HANDLE_PENDING_AND_DEOPT_IF": self.periodic_if, "ERROR_IF": self.error_if, "ERROR_NO_POP": self.error_no_pop, "DECREF_INPUTS": self.decref_inputs, @@ -171,6 +173,29 @@ def deopt_if( exit_if = deopt_if + def periodic_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + raise NotImplementedError("HANDLE_PENDING_AND_DEOPT_IF not support in tier 1") + + def exit_if_after( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + storage.clear_inputs("in AT_END_EXIT_IF") + storage.flush(self.out) + storage.stack.clear(self.out) + return self.exit_if(tkn, tkn_iter, uop, storage, inst) + def goto_error(self, offset: int, storage: Storage) -> str: if offset > 0: return f"JUMP_TO_LABEL(pop_{offset}_error);" @@ -692,6 +717,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_EVAL_BREAK_FLAG") if p.deopts: flags.append("HAS_DEOPT_FLAG") + if p.deopts_periodic: + flags.append("HAS_PERIODIC_FLAG") if p.side_exit: flags.append("HAS_EXIT_FLAG") if not p.infallible: diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index 72020133738fa5..29e4e74da72154 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -174,7 +174,13 @@ list of annotations and their meanings are as follows: * `override`. For external use by other interpreter definitions to override the current instruction definition. * `pure`. This instruction has no side effects. -* 'tierN'. This instruction is only used by the tier N interpreter. +* `tierN`. This instruction is only used by the tier N interpreter. +* `specializing`. A prefix for an instructions related to adaptive interpreter. +* `replaced`. This instruction will be replaced in the final bytecode by its directed + version (either forward or backward). +* `register`. Currently does nothing. +* `replicate(N)`. Replicate the instruction N times to store the oparg "inside" the instruction. +* `no_save_ip`. This instruction does not affect the instruction pointer. ### Special functions/macros diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 0bcdc5395dcd8e..b649b38123388d 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -53,10 +53,9 @@ "ESCAPES", "EXIT", "PURE", - "PASSTHROUGH", - "OPARG_AND_1", "ERROR_NO_POP", "NO_SAVE_IP", + "PERIODIC", ] diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 32dc346d5e891a..2fc6794f6a0887 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -132,7 +132,7 @@ def uses_this(inst: Instruction) -> bool: continue for tkn in uop.body.tokens(): if (tkn.kind == "IDENTIFIER" - and (tkn.text in {"DEOPT_IF", "EXIT_IF"})): + and (tkn.text in {"DEOPT_IF", "EXIT_IF", "AT_END_EXIT_IF"})): return True return False diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index fc3bc47286f7f6..1bb5f48658ddfc 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -111,6 +111,8 @@ def exit_if( self.emit("}\n") return not always_true(first_tkn) + periodic_if = deopt_if + def oparg( self, tkn: Token, diff --git a/Tools/i18n/msgfmt.py b/Tools/i18n/msgfmt.py index cd5f1ed9f3e268..65254a7c375456 100755 --- a/Tools/i18n/msgfmt.py +++ b/Tools/i18n/msgfmt.py @@ -5,8 +5,7 @@ This program converts a textual Uniforum-style message catalog (.po file) into a binary GNU catalog (.mo file). This is essentially the same function as the -GNU msgfmt program, however, it is a simpler implementation. Currently it -does not handle plural forms but it does handle message contexts. +GNU msgfmt program, however, it is a simpler implementation. Usage: msgfmt.py [OPTIONS] filename.po diff --git a/Tools/jit/template.c b/Tools/jit/template.c index d07f56e9ce6b42..7bbc44a08a0a98 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -51,7 +51,6 @@ do { \ OPT_STAT_INC(traces_executed); \ _PyExecutorObject *_executor = (EXECUTOR); \ - tstate->current_executor = (PyObject *)_executor; \ jit_func_preserve_none jitted = _executor->jit_side_entry; \ __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ } while (0) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 0beaab2d3e7157..73236767374378 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.16.1 +mypy==1.17.1 # needed for peg_generator: -types-psutil==7.0.0.20250601 -types-setuptools==80.9.0.20250529 +types-psutil==7.0.0.20250801 +types-setuptools==80.9.0.20250801 diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 93421b623b92f9..6bd31e8e6ecb9d 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -43,8 +43,5 @@ race:list_inplace_repeat_lock_held race:PyObject_Realloc # gh-133467. Some of these could be hard to trigger. -race_top:_Py_slot_tp_getattr_hook -race_top:slot_tp_descr_get -race_top:type_set_name race_top:set_tp_bases race_top:type_set_bases_unlocked diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index fdd93d5c0aa4af..973d78caa0849e 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -20,7 +20,8 @@ assert (CHECKOUT / "configure").is_file(), "Please update the location of the file" CROSS_BUILD_DIR = CHECKOUT / "cross-build" -BUILD_DIR = CROSS_BUILD_DIR / "build" +# Build platform can also be found via `config.guess`. +BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = ("# Generated by Tools/wasm/wasi .\n" @@ -80,7 +81,7 @@ def wrapper(context): print("📁", working_dir) if (clean_ok and getattr(context, "clean", False) and working_dir.exists()): - print(f"🚮 Deleting directory (--clean)...") + print("🚮 Deleting directory (--clean)...") shutil.rmtree(working_dir) working_dir.mkdir(parents=True, exist_ok=True) @@ -120,12 +121,6 @@ def call(command, *, context=None, quiet=False, logdir=None, **kwargs): subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) -def build_platform(): - """The name of the build/host platform.""" - # Can also be found via `config.guess`. - return sysconfig.get_config_var("BUILD_GNU_TYPE") - - def build_python_path(): """The path to the build Python binary.""" binary = BUILD_DIR / "python" @@ -274,7 +269,7 @@ def configure_wasi_python(context, working_dir): # executed from within a checkout. configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), f"--host={context.host_triple}", - f"--build={build_platform()}", + f"--build={BUILD_DIR.name}", f"--with-build-python={build_python}"] if build_python_is_pydebug(): configure.append("--with-pydebug") @@ -357,8 +352,8 @@ def main(): "Python)") make_host = subcommands.add_parser("make-host", help="Run `make` for the host/WASI") - clean = subcommands.add_parser("clean", help="Delete files and directories " - "created by this script") + subcommands.add_parser("clean", help="Delete files and directories " + "created by this script") for subcommand in build, configure_build, make_build, configure_host, make_host: subcommand.add_argument("--quiet", action="store_true", default=False, dest="quiet", diff --git a/configure b/configure index 0e7aefed5ee62d..451f72fdfd4ec3 100755 --- a/configure +++ b/configure @@ -725,8 +725,6 @@ LIBHACL_BLAKE2_SIMD128_OBJS LIBHACL_SIMD128_FLAGS LIBHACL_LDFLAGS LIBHACL_CFLAGS -LIBHASHLIB_INTERNAL -LIBHASHLIB_INTERNAL_CFLAGS MODULE_UNICODEDATA_FALSE MODULE_UNICODEDATA_TRUE MODULE__MULTIBYTECODEC_FALSE @@ -19039,15 +19037,27 @@ printf "%s\n" "#define WITH_DTRACE 1" >>confdefs.h # linked into the binary. Correspondingly, dtrace(1) is missing the ELF # generation flag '-G'. We check for presence of this flag, rather than # hardcoding support by OS, in the interest of robustness. + # + # NetBSD DTrace requires the -x nolibs flag to avoid system library conflicts + # and uses header generation for testing instead of object generation. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether DTrace probes require linking" >&5 printf %s "checking whether DTrace probes require linking... " >&6; } if test ${ac_cv_dtrace_link+y} then : printf %s "(cached) " >&6 else case e in #( - e) ac_cv_dtrace_link=no + e) + ac_cv_dtrace_link=no echo 'BEGIN{}' > conftest.d - "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ + case $host in + *netbsd*) + DTRACE_TEST_FLAGS="-x nolibs -h" + ;; + *) + DTRACE_TEST_FLAGS="-G" + ;; + esac + "$DTRACE" $DFLAGS $DTRACE_TEST_FLAGS -s conftest.d -o conftest.o > /dev/null 2>&1 && \ ac_cv_dtrace_link=yes ;; esac @@ -19057,6 +19067,12 @@ printf "%s\n" "$ac_cv_dtrace_link" >&6; } if test "$ac_cv_dtrace_link" = "yes"; then DTRACE_OBJS="Python/pydtrace.o" fi + # Set NetBSD-specific DTrace flags in DFLAGS + case $host in + *netbsd*) + DFLAGS="$DFLAGS -x nolibs" + ;; + esac fi PLATFORM_HEADERS= @@ -29951,7 +29967,6 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ - Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -32528,15 +32543,6 @@ then : fi -############################################################################### -# Cryptographic primitives -LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" -LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" -LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" - - - - ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -32785,8 +32791,8 @@ fi if test "x$py_cv_module__md5" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS)$as_nl" fi if test "$py_cv_module__md5" = yes; then @@ -32830,8 +32836,8 @@ fi if test "x$py_cv_module__sha1" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS)$as_nl" fi if test "$py_cv_module__sha1" = yes; then @@ -32875,8 +32881,8 @@ fi if test "x$py_cv_module__sha2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS)$as_nl" fi if test "$py_cv_module__sha2" = yes; then @@ -32920,8 +32926,8 @@ fi if test "x$py_cv_module__sha3" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS)$as_nl" fi if test "$py_cv_module__sha3" = yes; then @@ -32965,8 +32971,8 @@ fi if test "x$py_cv_module__blake2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS)$as_nl" fi if test "$py_cv_module__blake2" = yes; then @@ -33011,8 +33017,8 @@ fi if test "x$py_cv_module__hmac" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS)$as_nl" fi if test "$py_cv_module__hmac" = yes; then @@ -33693,8 +33699,8 @@ fi if test "x$py_cv_module__hashlib" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS$as_nl" fi if test "$py_cv_module__hashlib" = yes; then diff --git a/configure.ac b/configure.ac index 1e590e1d0fd727..f0e9eb2ee88c03 100644 --- a/configure.ac +++ b/configure.ac @@ -5104,16 +5104,33 @@ then # linked into the binary. Correspondingly, dtrace(1) is missing the ELF # generation flag '-G'. We check for presence of this flag, rather than # hardcoding support by OS, in the interest of robustness. + # + # NetBSD DTrace requires the -x nolibs flag to avoid system library conflicts + # and uses header generation for testing instead of object generation. AC_CACHE_CHECK([whether DTrace probes require linking], - [ac_cv_dtrace_link], [dnl + [ac_cv_dtrace_link], [ ac_cv_dtrace_link=no echo 'BEGIN{}' > conftest.d - "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ + case $host in + *netbsd*) + DTRACE_TEST_FLAGS="-x nolibs -h" + ;; + *) + DTRACE_TEST_FLAGS="-G" + ;; + esac + "$DTRACE" $DFLAGS $DTRACE_TEST_FLAGS -s conftest.d -o conftest.o > /dev/null 2>&1 && \ ac_cv_dtrace_link=yes ]) if test "$ac_cv_dtrace_link" = "yes"; then DTRACE_OBJS="Python/pydtrace.o" fi + # Set NetBSD-specific DTrace flags in DFLAGS + case $host in + *netbsd*) + DFLAGS="$DFLAGS -x nolibs" + ;; + esac fi dnl Platform-specific C and header files. @@ -5219,7 +5236,7 @@ AC_CHECK_FUNCS([ \ posix_spawn_file_actions_addclosefrom_np \ pread preadv preadv2 process_vm_readv \ pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ - pthread_kill pthread_get_name_np pthread_getname_np pthread_set_name_np + pthread_kill pthread_get_name_np pthread_getname_np pthread_set_name_np \ pthread_setname_np pthread_getattr_np \ ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ @@ -7187,7 +7204,6 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ - Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -7958,15 +7974,6 @@ PY_STDLIB_MOD_SIMPLE([_codecs_tw]) PY_STDLIB_MOD_SIMPLE([_multibytecodec]) PY_STDLIB_MOD_SIMPLE([unicodedata]) -############################################################################### -# Cryptographic primitives -LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" -LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" -LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" - -AC_SUBST([LIBHASHLIB_INTERNAL_CFLAGS]) -AC_SUBST([LIBHASHLIB_INTERNAL]) - ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -8103,9 +8110,7 @@ dnl The EXTNAME is the name of the extension module being built. AC_DEFUN([PY_HACL_CREATE_MODULE], [ AS_VAR_PUSHDEF([v], [[LIBHACL_][$1][_LDFLAGS]]) AS_VAR_SET([v], [[LIBHACL_][$1][_LIB_${LIBHACL_LDEPS_LIBTYPE}]]) - PY_STDLIB_MOD([$2], [$3], [], - [$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS], - [\$($v) $LIBHASHLIB_INTERNAL_LDFLAGS]) + PY_STDLIB_MOD([$2], [$3], [], [$LIBHACL_CFLAGS], [\$($v)]) AS_VAR_POPDEF([v]) ]) @@ -8186,8 +8191,7 @@ dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], - [$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS], - [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS]) + [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) dnl test modules PY_STDLIB_MOD([_testcapi], pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy