From 94cde64342fa8c7cd55a6d2b60f1e57654220bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 10 Jun 2025 13:42:25 +0200 Subject: [PATCH 01/13] faster CI --- .github/workflows/build.yml | 283 +----------------------------------- .github/workflows/lint.yml | 28 ---- 2 files changed, 5 insertions(+), 306 deletions(-) delete mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54ebc914b46821..6312530669623e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,12 +43,6 @@ jobs: # uses: ./.github/workflows/reusable-context.yml - check-docs: - name: Docs - needs: build-context - if: fromJSON(needs.build-context.outputs.run-docs) - uses: ./.github/workflows/reusable-docs.yml - check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job @@ -160,15 +154,12 @@ jobs: needs: build-context if: fromJSON(needs.build-context.outputs.run-windows-tests) strategy: - fail-fast: false + fail-fast: true matrix: arch: - x64 - - Win32 - - arm64 free-threading: - false - - true exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } @@ -177,22 +168,6 @@ jobs: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} - build-windows-msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} - needs: build-context - if: fromJSON(needs.build-context.outputs.run-windows-msi) - strategy: - fail-fast: false - matrix: - arch: - - x86 - - x64 - - arm64 - uses: ./.github/workflows/reusable-windows-msi.yml - with: - arch: ${{ matrix.arch }} - build-macos: name: >- macOS @@ -200,20 +175,17 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: false + fail-fast: true matrix: # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. # macOS 13 only runs tests against the GIL-enabled CPython. # Cirrus used for upstream, macos-14 for forks. os: - ghcr.io/cirruslabs/macos-runner:sonoma - - macos-14 - - macos-13 is-fork: # only used for the exclusion trick - ${{ github.repository_owner != 'python' }} free-threading: - false - - true exclude: - os: ghcr.io/cirruslabs/macos-runner:sonoma is-fork: true @@ -235,17 +207,14 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: false + fail-fast: true matrix: bolt: - false - - true free-threading: - false - - true os: - ubuntu-24.04 - - ubuntu-24.04-arm exclude: # Do not test BOLT with free-threading, to conserve resources - bolt: true @@ -267,10 +236,10 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: false + fail-fast: true matrix: os: [ubuntu-24.04] - openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] + openssl_ver: [3.0.16] # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} @@ -330,203 +299,6 @@ jobs: with: config_hash: ${{ needs.build-context.outputs.config-hash }} - test-hypothesis: - name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-24.04 - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - env: - OPENSSL_VER: 3.0.16 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" - echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - - name: Create directories for read-only out-of-tree builds - run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - - name: Bind mount sources read-only - run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --enable-slower-safety \ - --with-openssl="$OPENSSL_DIR" - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - - name: "Create hypothesis venv" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - VENV_LOC=$(realpath -m .)/hypovenv - VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" - echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" - ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - - name: 'Restore Hypothesis database' - id: cache-hypothesis-database - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ - key: hypothesis-database-${{ github.head_ref || github.run_id }} - restore-keys: | - hypothesis-database- - - name: "Run tests" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - # Most of the excluded tests are slow test suites with no property tests - # - # (GH-104097) test_sysconfig is skipped because it has tests that are - # failing when executed from inside a virtual environment. - "${VENV_PYTHON}" -m test \ - -W \ - --slowest \ - -j4 \ - --timeout 900 \ - -x test_asyncio \ - -x test_multiprocessing_fork \ - -x test_multiprocessing_forkserver \ - -x test_multiprocessing_spawn \ - -x test_concurrent_futures \ - -x test_socket \ - -x test_subprocess \ - -x test_signal \ - -x test_sysconfig - - uses: actions/upload-artifact@v4 - if: always() - with: - name: hypothesis-example-db - path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ - - build-asan: - name: 'Address sanitizer' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - env: - OPENSSL_VER: 3.0.16 - PYTHONSTRICTEXTENSIONBUILD: 1 - ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 10 - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: xvfb-run make ci - - build-tsan: - name: >- - Thread sanitizer - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - free-threading: - - false - - true - uses: ./.github/workflows/reusable-tsan.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - free-threading: ${{ matrix.free-threading }} - cross-build-linux: name: Cross build Linux runs-on: ubuntu-latest @@ -567,45 +339,6 @@ jobs: run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ - cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-ci-fuzz == 'true' - permissions: - security-events: write - strategy: - fail-fast: false - matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 - oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass @@ -613,20 +346,14 @@ jobs: timeout-minutes: 5 needs: - build-context # Transitive dependency, needed to access `run-tests` value - - check-docs - check-autoconf-regen - check-generated-files - build-windows - - build-windows-msi - build-macos - build-ubuntu - build-ubuntu-ssltests - build-wasi - - test-hypothesis - - build-asan - - build-tsan - cross-build-linux - - cifuzz if: always() steps: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index d74ce8fcc256dc..00000000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Lint - -on: [push, pull_request, workflow_dispatch] - -permissions: - contents: read - -env: - FORCE_COLOR: 1 - RUFF_OUTPUT_FORMAT: github - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - lint: - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - uses: pre-commit/action@v3.0.1 From 3dd0a97801d707a7312910ad37f0bcb2cc139a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 10 Jun 2025 12:56:07 +0200 Subject: [PATCH 02/13] extract `_hashlib` helpers into a separate directory --- Makefile.pre.in | 41 ++++++--- Modules/_hashlib/hashlib_buffer.c | 40 +++++++++ Modules/_hashlib/hashlib_buffer.h | 60 +++++++++++++ Modules/_hashlib/hashlib_fetch.c | 1 + Modules/_hashlib/hashlib_fetch.h | 130 +++++++++++++++++++++++++++ Modules/_hashlib/hashlib_mutex.h | 62 +++++++++++++ Modules/_hashopenssl.c | 13 ++- Modules/blake2module.c | 11 ++- Modules/hashlib.h | 116 ------------------------ Modules/hmacmodule.c | 5 +- Modules/md5module.c | 11 ++- Modules/sha1module.c | 11 ++- Modules/sha2module.c | 13 ++- Modules/sha3module.c | 10 +-- PCbuild/_hashlib.vcxproj | 6 ++ PCbuild/_hashlib.vcxproj.filters | 2 +- PCbuild/pythoncore.vcxproj | 4 + PCbuild/pythoncore.vcxproj.filters | 15 ++++ Tools/c-analyzer/cpython/ignored.tsv | 1 + configure | 35 +++++--- configure.ac | 14 ++- 21 files changed, 417 insertions(+), 184 deletions(-) create mode 100644 Modules/_hashlib/hashlib_buffer.c create mode 100644 Modules/_hashlib/hashlib_buffer.h create mode 100644 Modules/_hashlib/hashlib_fetch.c create mode 100644 Modules/_hashlib/hashlib_fetch.h create mode 100644 Modules/_hashlib/hashlib_mutex.h delete mode 100644 Modules/hashlib.h diff --git a/Makefile.pre.in b/Makefile.pre.in index b5703fbe6ae974..ba681d92e46465 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1481,6 +1481,12 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) +########################################################################## +# '_hashlib', '_hmac' and HACL*-based modules helpers + +Modules/_hashlib/hashlib_buffer.o: $(srcdir)/Modules/_hashlib/hashlib_buffer.c $(srcdir)/Modules/_hashlib/hashlib_buffer.h $(PYTHON_HEADERS) + $(CC) -I$(srcdir)/Modules/_hashlib -c $(PY_STDMODULE_CFLAGS) $(CCSHARED) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c + ########################################################################## # HACL* library build # @@ -3323,22 +3329,31 @@ MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ -MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h +MODULE__HASHLIB_DEPS= \ + $(srcdir)/Modules/_hashlib/hashlib_buffer.h \ + $(srcdir)/Modules/_hashlib/hashlib_fetch.h \ + $(srcdir)/Modules/_hashlib/hashlib_mutex.h + +MODULE__HASHLIB_LDEPS= \ + Modules/_hashlib/hashlib_buffer.o \ + Modules/_hashlib/hashlib_fetch.o + # HACL*-based cryptographic primitives -MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__MD5_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA1_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA1_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA2_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA3_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA3_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__BLAKE2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__BLAKE2_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) + +MODULE__HMAC_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__HMAC_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h diff --git a/Modules/_hashlib/hashlib_buffer.c b/Modules/_hashlib/hashlib_buffer.c new file mode 100644 index 00000000000000..34811c6266fd64 --- /dev/null +++ b/Modules/_hashlib/hashlib_buffer.c @@ -0,0 +1,40 @@ +#include "hashlib_buffer.h" + +int +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) +{ + if (data != NULL && string == NULL) { + // called as H(data) or H(data=...) + *res = data; + return 1; + } + else if (data == NULL && string != NULL) { + // called as H(string=...) + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead", 1) < 0) + { + *res = NULL; + return -1; + } + *res = string; + return 1; + } + else if (data == NULL && string == NULL) { + // fast path when no data is given + assert(!PyErr_Occurred()); + *res = NULL; + return 0; + } + else { + // called as H(data=..., string) + *res = NULL; + PyErr_SetString(PyExc_TypeError, + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version."); + return -1; + } +} diff --git a/Modules/_hashlib/hashlib_buffer.h b/Modules/_hashlib/hashlib_buffer.h new file mode 100644 index 00000000000000..13b0aa1ab3cb61 --- /dev/null +++ b/Modules/_hashlib/hashlib_buffer.h @@ -0,0 +1,60 @@ +#ifndef _HASHLIB_HASHLIB_BUFFER_H +#define _HASHLIB_HASHLIB_BUFFER_H + +#include "Python.h" + +/* + * Given an buffer-like OBJ, fill in the buffer VIEW with the result + * of PyObject_GetBuffer. + * + * On error, set an exception and execute the ERRACTION statements, + * e.g. 'return NULL' or 'goto error'. + * + * Parameters + * + * OBJ An object supporting the buffer API. + * VIEW A Py_buffer pointer to fill. + * ERRACTION The statements to execute on error. + */ +#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION) \ + do { \ + if (PyUnicode_Check((OBJ))) { \ + PyErr_SetString(PyExc_TypeError, \ + "strings must be encoded before hashing"); \ + ERRACTION; \ + } \ + if (!PyObject_CheckBuffer((OBJ))) { \ + PyErr_SetString(PyExc_TypeError, \ + "object supporting the buffer API required"); \ + ERRACTION; \ + } \ + if (PyObject_GetBuffer((OBJ), (VIEW), PyBUF_SIMPLE) == -1) { \ + ERRACTION; \ + } \ + if ((VIEW)->ndim > 1) { \ + PyErr_SetString(PyExc_BufferError, \ + "buffer must be one-dimensional"); \ + PyBuffer_Release((VIEW)); \ + ERRACTION; \ + } \ + } while(0) + +/* Specialization of GET_BUFFER_VIEW_OR_ERROR() returning NULL on error. */ +#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW) \ + GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL) + +/* + * Allow to use the 'data' or 'string' keyword in hashlib.new() + * and other hash functions named constructors. + * + * - If 'data' and 'string' are both non-NULL, set an exception and return -1. + * - If 'data' and 'string' are both NULL, set '*res' to NULL and return 0. + * - Otherwise, set '*res' to 'data' or 'string' and return 1. A deprecation + * warning is set when 'string' is specified. + * + * The symbol is exported for '_hashlib' and HACL*-based extension modules. + */ +PyAPI_FUNC(int) +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string); + +#endif // !_HASHLIB_HASHLIB_BUFFER_H diff --git a/Modules/_hashlib/hashlib_fetch.c b/Modules/_hashlib/hashlib_fetch.c new file mode 100644 index 00000000000000..6b772030ad87d0 --- /dev/null +++ b/Modules/_hashlib/hashlib_fetch.c @@ -0,0 +1 @@ +#include "hashlib_fetch.h" diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h new file mode 100644 index 00000000000000..ee647ea6220f4d --- /dev/null +++ b/Modules/_hashlib/hashlib_fetch.h @@ -0,0 +1,130 @@ +/* + * Interface for fetching a message digest from a digest-like identifier. + * + * The following table summaries the possible algorthms: + * + * +----------+--------------+--------------+---------------------------------+ + * | Family | Algorithm | Python Name | Notes | + * +==========+==============+==============+=================================+ + * | MD @ | + * | +--------------+--------------+---------------------------------+ + * | | MD5 | "md5" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA1 @ | + * | +--------------+--------------+---------------------------------+ + * | | SHA1-160 | "sha1" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA2 @ | + * | +--------------+--------------+---------------------------------+ + * | | SHA2-224 | "sha224" | | + * | | SHA2-256 | "sha256" | | + * | | SHA2-384 | "sha384" | | + * | | SHA2-512 | "sha512" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA2t @ Truncated SHA2-512 | + * | +--------------+--------------+---------------------------------+ + * | | SHA2-512/224 | "sha512_224" | | + * | | SHA2-512/256 | "sha512_256" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA3 @ | + * | +--------------+--------------+---------------------------------+ + * | | SHA3-224 | "sha3_224" | | + * | | SHA3-256 | "sha3_256" | | + * | | SHA3-384 | "sha3_384" | | + * | | SHA3-512 | "sha3_512" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA3-XOF @ Extensible Output Functions | + * | +--------------+--------------+---------------------------------+ + * | | SHAKE-128 | "shake_128" | | + * | | SHAKE-256 | "shake_256" | | + * +----------+--------------+--------------+---------------------------------+ + * | BLAKE2 @ | + * | +--------------+--------------+---------------------------------+ + * | | BLAKE2b | "blake2b" | | + * | | BLAKE2s | "blake2s" | | + * +----------+--------------+--------------+---------------------------------+ + */ + +#ifndef _HASHLIB_HASHLIB_FETCH_H +#define _HASHLIB_HASHLIB_FETCH_H + +#include "Python.h" + +#define Py_HASHLIB_MD_NS(ATTR) Py_hashlib_message_digest_ ## ATTR +#define Py_HASHLIB_MD_FAMILY(FAMILY_ID) Py_HASHLIB_MD_NS(family_ ## FAMILY_ID) +#define Py_HASHLIB_MD_MEMBER(MEMBER_ID) Py_HASHLIB_MD_NS(member_ ## MEMBER_ID) + +#define Py_HASHLIB_MD_NAMES Py_HASHLIB_MD_NS(NAMES) +#define Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(Py_HASHLIB_MD_NAMES) +#define Py_HASHLIB_MD_NAME(MEMBER_ID) \ + ( \ + assert(Py_HASHLIB_MD_NAME(MEMBER_ID) < Py_HASHLIB_MD_COUNT), \ + Py_HASHLIB_MD_NAMES[Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ + ) + +typedef enum { + Py_HASHLIB_MD_FAMILY(MD) = 0, + Py_HASHLIB_MD_FAMILY(SHA1), + Py_HASHLIB_MD_FAMILY(SHA2), + Py_HASHLIB_MD_FAMILY(SHA2t), + Py_HASHLIB_MD_FAMILY(SHA3), + Py_HASHLIB_MD_FAMILY(SHA3_XOF), + Py_HASHLIB_MD_FAMILY(BLAKE2), +} Py_HASHLIB_MD_NS(family); + +typedef enum { + /* MD-family */ + Py_HASHLIB_MD_MEMBER(md5) = 0, + /* SHA-1 family */ + Py_HASHLIB_MD_MEMBER(sha1), + /* SHA-2 family */ + Py_HASHLIB_MD_MEMBER(sha224), + Py_HASHLIB_MD_MEMBER(sha256), + Py_HASHLIB_MD_MEMBER(sha384), + Py_HASHLIB_MD_MEMBER(sha512), + /* Truncated SHA-2 family */ + Py_HASHLIB_MD_MEMBER(sha512_224), + Py_HASHLIB_MD_MEMBER(sha512_256), + /* SHA-3 family */ + Py_HASHLIB_MD_MEMBER(sha3_224), + Py_HASHLIB_MD_MEMBER(sha3_256), + Py_HASHLIB_MD_MEMBER(sha3_384), + Py_HASHLIB_MD_MEMBER(sha3_512), + /* SHA-3 XOF SHAKE family */ + Py_HASHLIB_MD_MEMBER(shake_128), + Py_HASHLIB_MD_MEMBER(shake_256), + /* BLAKE-2 family */ + Py_HASHLIB_MD_MEMBER(blake2b), + Py_HASHLIB_MD_MEMBER(blake2s), +} Py_HASHLIB_MD_NS(member); + +static const char *Py_HASHLIB_MD_NAMES[] = { +#define DECL_MESSAGE_DIGEST_NAME(ID) [Py_HASHLIB_MD_MEMBER(ID)] = #ID + /* MD-family */ + DECL_MESSAGE_DIGEST_NAME(md5), + /* SHA-1 family */ + DECL_MESSAGE_DIGEST_NAME(sha1), + /* SHA-2 family */ + DECL_MESSAGE_DIGEST_NAME(sha224), + DECL_MESSAGE_DIGEST_NAME(sha256), + DECL_MESSAGE_DIGEST_NAME(sha384), + DECL_MESSAGE_DIGEST_NAME(sha512), + /* Truncated SHA-2 family */ + DECL_MESSAGE_DIGEST_NAME(sha512_224), + DECL_MESSAGE_DIGEST_NAME(sha512_256), + /* SHA-3 family */ + DECL_MESSAGE_DIGEST_NAME(sha3_224), + DECL_MESSAGE_DIGEST_NAME(sha3_256), + DECL_MESSAGE_DIGEST_NAME(sha3_384), + DECL_MESSAGE_DIGEST_NAME(sha3_512), + /* SHA-3 XOF SHAKE family */ + DECL_MESSAGE_DIGEST_NAME(shake_128), + DECL_MESSAGE_DIGEST_NAME(shake_256), + /* BLAKE-2 family */ + DECL_MESSAGE_DIGEST_NAME(blake2b), + DECL_MESSAGE_DIGEST_NAME(blake2s), +#undef DECL_MESSAGE_DIGEST_NAME + NULL /* sentinel */ +}; + +#endif // !_HASHLIB_HASHLIB_FETCH_H diff --git a/Modules/_hashlib/hashlib_mutex.h b/Modules/_hashlib/hashlib_mutex.h new file mode 100644 index 00000000000000..cba7524f0cc668 --- /dev/null +++ b/Modules/_hashlib/hashlib_mutex.h @@ -0,0 +1,62 @@ +#ifndef _HASHLIB_HASHLIB_MUTEX_H +#define _HASHLIB_HASHLIB_MUTEX_H + +#include "Python.h" +#include "pycore_lock.h" // PyMutex + +/* + * Maximum number of bytes for a message for which the GIL is held + * when performing incremental hashing. + */ +#define HASHLIB_GIL_MINSIZE 2048 + +/* + * Helper code to synchronize access to the hash object when the GIL is + * released around a CPU consuming hashlib operation. All code paths that + * access a mutable part of obj must be enclosed in an ENTER_HASHLIB / + * LEAVE_HASHLIB block or explicitly acquire and release the lock inside + * a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for + * an operation. + * + * These only drop the GIL if the lock acquisition itself is likely to + * block. Thus the non-blocking acquire gating the GIL release for a + * blocking lock acquisition. The intent of these macros is to surround + * the assumed always "fast" operations that you aren't releasing the + * GIL around. Otherwise use code similar to what you see in hash + * function update() methods. + */ + +/* Prevent undefined behaviors via multiple threads entering the C API. */ +#define HASHLIB_LOCK_HEAD \ + bool use_mutex; \ + PyMutex mutex; + +#ifdef Py_GIL_DISABLED +#define HASHLIB_INIT_MUTEX(OBJ) \ + do { \ + (OBJ)->mutex = (PyMutex){0}; \ + (OBJ)->use_mutex = true; \ + } while (0) +#else +#define HASHLIB_INIT_MUTEX(OBJ) \ + do { \ + (OBJ)->mutex = (PyMutex){0}; \ + (OBJ)->use_mutex = false; \ + } while (0) +#endif + +#define ENTER_HASHLIB(OBJ) \ + do { \ + if ((OBJ)->use_mutex) { \ + PyMutex_Lock(&(OBJ)->mutex); \ + } \ + } while (0) + +#define LEAVE_HASHLIB(OBJ) \ + do { \ + if ((OBJ)->use_mutex) { \ + PyMutex_Unlock(&(OBJ)->mutex); \ + } \ + } while (0) + +#endif // !_HASHLIB_HASHLIB_MUTEX_H diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index ce9603d5db841f..8d3db312709502 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -26,7 +26,8 @@ #include "pycore_hashtable.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED -#include "hashlib.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" /* EVP is the preferred interface to hashing in OpenSSL */ #include @@ -279,20 +280,16 @@ get_hashlib_state(PyObject *module) typedef struct { PyObject_HEAD + HASHLIB_LOCK_HEAD EVP_MD_CTX *ctx; /* OpenSSL message digest context */ - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; /* OpenSSL context lock */ } HASHobject; #define HASHobject_CAST(op) ((HASHobject *)(op)) typedef struct { PyObject_HEAD + HASHLIB_LOCK_HEAD HMAC_CTX *ctx; /* OpenSSL hmac context */ - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; /* HMAC context lock */ } HMACobject; #define HMACobject_CAST(op) ((HMACobject *)(op)) @@ -1126,7 +1123,7 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, if (view.buf && view.len) { if (view.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor + /* Do not initialize self->mutex here as this is the constructor * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS result = _hashlib_HASH_hash(self, view.buf, view.len); diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 2ce8c0cd3d7b6f..84b499dbacb2c7 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -2,6 +2,7 @@ * Written in 2013 by Dmitry Chestnykh * Modified for CPython by Christian Heimes * Updated to use HACL* by Jonathan Protzenko + * Refactored by Bénédikt Tran <10796600+picnixz@users.noreply.github.com> * * To the extent possible under law, the author have dedicated all * copyright and related and neighboring rights to this software to @@ -14,10 +15,11 @@ #endif #include "Python.h" -#include "hashlib.h" +#include "pycore_moduleobject.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" -#include "pycore_moduleobject.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" // QUICK CPU AUTODETECTION // @@ -353,6 +355,7 @@ type_to_impl(PyTypeObject *type) typedef struct { PyObject_HEAD + HASHLIB_LOCK_HEAD union { Hacl_Hash_Blake2s_state_t *blake2s_state; Hacl_Hash_Blake2b_state_t *blake2b_state; @@ -364,8 +367,6 @@ typedef struct { #endif }; blake2_impl impl; - bool use_mutex; - PyMutex mutex; } Blake2Object; #define _Blake2Object_CAST(op) ((Blake2Object *)(op)) @@ -647,6 +648,8 @@ py_blake2_new(PyTypeObject *type, PyObject *data, int digest_size, Py_buffer buf; GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* Do not initialize self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS update(self, buf.buf, buf.len); Py_END_ALLOW_THREADS diff --git a/Modules/hashlib.h b/Modules/hashlib.h deleted file mode 100644 index e82ec92be25c57..00000000000000 --- a/Modules/hashlib.h +++ /dev/null @@ -1,116 +0,0 @@ -/* Common code for use by all hashlib related modules. */ - -#include "pycore_lock.h" // PyMutex - -/* - * Given a PyObject* obj, fill in the Py_buffer* viewp with the result - * of PyObject_GetBuffer. Sets an exception and issues the erraction - * on any errors, e.g. 'return NULL' or 'goto error'. - */ -#define GET_BUFFER_VIEW_OR_ERROR(obj, viewp, erraction) do { \ - if (PyUnicode_Check((obj))) { \ - PyErr_SetString(PyExc_TypeError, \ - "Strings must be encoded before hashing");\ - erraction; \ - } \ - if (!PyObject_CheckBuffer((obj))) { \ - PyErr_SetString(PyExc_TypeError, \ - "object supporting the buffer API required"); \ - erraction; \ - } \ - if (PyObject_GetBuffer((obj), (viewp), PyBUF_SIMPLE) == -1) { \ - erraction; \ - } \ - if ((viewp)->ndim > 1) { \ - PyErr_SetString(PyExc_BufferError, \ - "Buffer must be single dimension"); \ - PyBuffer_Release((viewp)); \ - erraction; \ - } \ - } while(0) - -#define GET_BUFFER_VIEW_OR_ERROUT(obj, viewp) \ - GET_BUFFER_VIEW_OR_ERROR(obj, viewp, return NULL) - -/* - * Helper code to synchronize access to the hash object when the GIL is - * released around a CPU consuming hashlib operation. All code paths that - * access a mutable part of obj must be enclosed in an ENTER_HASHLIB / - * LEAVE_HASHLIB block or explicitly acquire and release the lock inside - * a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for - * an operation. - * - * These only drop the GIL if the lock acquisition itself is likely to - * block. Thus the non-blocking acquire gating the GIL release for a - * blocking lock acquisition. The intent of these macros is to surround - * the assumed always "fast" operations that you aren't releasing the - * GIL around. Otherwise use code similar to what you see in hash - * function update() methods. - */ - -#include "pythread.h" -#define ENTER_HASHLIB(obj) \ - if ((obj)->use_mutex) { \ - PyMutex_Lock(&(obj)->mutex); \ - } -#define LEAVE_HASHLIB(obj) \ - if ((obj)->use_mutex) { \ - PyMutex_Unlock(&(obj)->mutex); \ - } - -#ifdef Py_GIL_DISABLED -#define HASHLIB_INIT_MUTEX(obj) \ - do { \ - (obj)->mutex = (PyMutex){0}; \ - (obj)->use_mutex = true; \ - } while (0) -#else -#define HASHLIB_INIT_MUTEX(obj) \ - do { \ - (obj)->mutex = (PyMutex){0}; \ - (obj)->use_mutex = false; \ - } while (0) -#endif - -/* TODO(gpshead): We should make this a module or class attribute - * to allow the user to optimize based on the platform they're using. */ -#define HASHLIB_GIL_MINSIZE 2048 - -static inline int -_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) -{ - if (data != NULL && string == NULL) { - // called as H(data) or H(data=...) - *res = data; - return 1; - } - else if (data == NULL && string != NULL) { - // called as H(string=...) - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "the 'string' keyword parameter is deprecated since " - "Python 3.15 and slated for removal in Python 3.19; " - "use the 'data' keyword parameter or pass the data " - "to hash as a positional argument instead", 1) < 0) - { - *res = NULL; - return -1; - } - *res = string; - return 1; - } - else if (data == NULL && string == NULL) { - // fast path when no data is given - assert(!PyErr_Occurred()); - *res = NULL; - return 0; - } - else { - // called as H(data=..., string) - *res = NULL; - PyErr_SetString(PyExc_TypeError, - "'data' and 'string' are mutually exclusive " - "and support for 'string' keyword parameter " - "is slated for removal in a future version."); - return -1; - } -} diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index b404d5732ec857..6c6593f9522bee 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -44,9 +44,10 @@ #include "_hacl/Hacl_Streaming_HMAC.h" // Hacl_Agile_Hash_* identifiers #include "_hacl/Hacl_Streaming_Types.h" // Hacl_Streaming_Types_error_code -#include +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" -#include "hashlib.h" +#include // --- Reusable error messages ------------------------------------------------ diff --git a/Modules/md5module.c b/Modules/md5module.c index 08dbcd2cbce844..7fe956b9f5fc3b 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -8,6 +8,7 @@ Andrew Kuchling (amk@amk.ca) Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) + Bénédikt Tran (10796600+picnixz@users.noreply.github.com) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. @@ -21,7 +22,8 @@ #endif #include "Python.h" -#include "hashlib.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" /*[clinic input] module _md5 @@ -36,12 +38,9 @@ class MD5Type "MD5object *" "&PyType_Type" #include "_hacl/Hacl_Hash_MD5.h" - typedef struct { PyObject_HEAD - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; + HASHLIB_LOCK_HEAD Hacl_Hash_MD5_state_t *hash_state; } MD5object; @@ -320,7 +319,7 @@ _md5_md5_impl(PyObject *module, PyObject *data, int usedforsecurity, if (string) { if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor + /* Do not initialize self->mutex here as this is the constructor * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS update(new->hash_state, buf.buf, buf.len); diff --git a/Modules/sha1module.c b/Modules/sha1module.c index a746bf74f8d4c1..6a0fbcd99c28cc 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -8,6 +8,7 @@ Andrew Kuchling (amk@amk.ca) Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) + Bénédikt Tran (10796600+picnixz@users.noreply.github.com) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. @@ -20,9 +21,10 @@ #endif #include "Python.h" -#include "hashlib.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" /*[clinic input] module _sha1 @@ -39,10 +41,7 @@ class SHA1Type "SHA1object *" "&PyType_Type" typedef struct { PyObject_HEAD - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; - PyThread_type_lock lock; + HASHLIB_LOCK_HEAD Hacl_Hash_SHA1_state_t *hash_state; } SHA1object; @@ -315,7 +314,7 @@ _sha1_sha1_impl(PyObject *module, PyObject *data, int usedforsecurity, } if (string) { if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor + /* Do not initialize self->mutex here as this is the constructor * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS update(new->hash_state, buf.buf, buf.len); diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 72931910c5d720..e3e5bde88e6992 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -9,6 +9,7 @@ Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) Jonathan Protzenko (jonathan@protzenko.fr) + Bénédikt Tran (10796600+picnixz@users.noreply.github.com) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. @@ -23,10 +24,10 @@ #include "Python.h" #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "pycore_strhex.h" // _Py_strhex() - -#include "hashlib.h" +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" /*[clinic input] module _sha2 @@ -51,10 +52,8 @@ class SHA512Type "SHA512object *" "&PyType_Type" typedef struct { PyObject_HEAD + HASHLIB_LOCK_HEAD int digestsize; - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; Hacl_Hash_SHA2_state_t_256 *state; } SHA256object; @@ -639,7 +638,7 @@ _sha2_sha256_impl(PyObject *module, PyObject *data, int usedforsecurity, } if (string) { if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor + /* Do not initialize self->mutex here as this is the constructor * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS update_256(new->state, buf.buf, buf.len); diff --git a/Modules/sha3module.c b/Modules/sha3module.c index cfbf0cbcc042c5..202de3eb690f73 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -9,6 +9,7 @@ * Greg Stein (gstein@lyra.org) * Trevor Perrin (trevp@trevp.net) * Gregory P. Smith (greg@krypto.org) + * Bénédikt Tran (10796600+picnixz@users.noreply.github.com) * * Copyright (C) 2012-2022 Christian Heimes (christian@python.org) * Licensed to PSF under a Contributor Agreement. @@ -22,7 +23,8 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "hashlib.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" #define SHA3_MAX_DIGESTSIZE 64 /* 64 Bytes (512 Bits) for 224 to 512 */ @@ -60,9 +62,7 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type" typedef struct { PyObject_HEAD - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; + HASHLIB_LOCK_HEAD Hacl_Hash_SHA3_state_t *hash_state; } SHA3object; @@ -164,7 +164,7 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data_obj, int usedforsecurity, if (data) { GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor + /* Do not initialize self->mutex here as this is the constructor * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS sha3_update(self->hash_state, buf.buf, buf.len); diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index 2cd205224bc089..9e6dcee40793d3 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -100,6 +100,12 @@ + + + + + + diff --git a/PCbuild/_hashlib.vcxproj.filters b/PCbuild/_hashlib.vcxproj.filters index 7a0700c007f644..d465d92a956eda 100644 --- a/PCbuild/_hashlib.vcxproj.filters +++ b/PCbuild/_hashlib.vcxproj.filters @@ -18,4 +18,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 32a8f2dbad3d5e..5695923019d218 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -443,6 +443,10 @@ HACL_CAN_COMPILE_VEC128;%(PreprocessorDefinitions) /arch:AVX %(AdditionalOptions) + + + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0e6d42cc959ba5..ad3fc157ff6308 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -255,6 +255,15 @@ Include + + Modules + + + Modules + + + Modules + Modules @@ -971,6 +980,12 @@ Modules + + Modules + + + Modules + Modules diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 15b18f5286b399..f3154f5a70ffda 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -236,6 +236,7 @@ Modules/_decimal/_decimal.c - signal_map_template - Modules/_decimal/_decimal.c - ssize_constants - Modules/_decimal/_decimal.c - INVALID_SIGNALDICT_ERROR_MSG - Modules/_elementtree.c - ExpatMemoryHandler - +Modules/_hashlib/hashlib_fetch.h - Py_hashlib_message_digest_NAMES - Modules/_hashopenssl.c - py_hashes - Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - Modules/_hacl/Hacl_Hash_MD5.c - _h0 - diff --git a/configure b/configure index 029bf527da4e3d..ae87370302defd 100755 --- a/configure +++ b/configure @@ -29970,6 +29970,7 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ + Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -32548,6 +32549,12 @@ then : fi +############################################################################### +# Cryptographic primitives +LIBHASHLIB_INCL='-I$(srcdir)/Modules/_hashlib' +LIBHASHLIB_LIBS='Modules/_hashlib/hashlib_buffer.o Modules/_hashlib/hashlib_fetch.o' + + ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -32796,8 +32803,8 @@ fi if test "x$py_cv_module__md5" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__md5" = yes; then @@ -32841,8 +32848,8 @@ fi if test "x$py_cv_module__sha1" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__sha1" = yes; then @@ -32886,8 +32893,8 @@ fi if test "x$py_cv_module__sha2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__sha2" = yes; then @@ -32931,8 +32938,8 @@ fi if test "x$py_cv_module__sha3" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__sha3" = yes; then @@ -32976,8 +32983,8 @@ fi if test "x$py_cv_module__blake2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__blake2" = yes; then @@ -33022,8 +33029,8 @@ fi if test "x$py_cv_module__hmac" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__hmac" = yes; then @@ -33692,8 +33699,8 @@ fi if test "x$py_cv_module__hashlib" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__hashlib" = yes; then diff --git a/configure.ac b/configure.ac index 371b2e8ed73525..f180e4ca58e805 100644 --- a/configure.ac +++ b/configure.ac @@ -7199,6 +7199,7 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ + Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -7971,6 +7972,12 @@ PY_STDLIB_MOD_SIMPLE([_codecs_tw]) PY_STDLIB_MOD_SIMPLE([_multibytecodec]) PY_STDLIB_MOD_SIMPLE([unicodedata]) +############################################################################### +# Cryptographic primitives +LIBHASHLIB_INCL='-I$(srcdir)/Modules/_hashlib' +LIBHASHLIB_LIBS='Modules/_hashlib/hashlib_buffer.o Modules/_hashlib/hashlib_fetch.o' + + ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -8105,7 +8112,9 @@ dnl The EXTNAME is the name of the extension module being built. AC_DEFUN([PY_HACL_CREATE_MODULE], [ AS_VAR_PUSHDEF([v], [[LIBHACL_][$1][_LDFLAGS]]) AS_VAR_SET([v], [[LIBHACL_][$1][_LIB_${LIBHACL_LDEPS_LIBTYPE}]]) - PY_STDLIB_MOD([$2], [$3], [], [$LIBHACL_CFLAGS], [\$($v)]) + PY_STDLIB_MOD([$2], [$3], [], + [$LIBHACL_CFLAGS $LIBHASHLIB_INCL], + [\$($v) $LIBHASHLIB_LIBS]) AS_VAR_POPDEF([v]) ]) @@ -8176,7 +8185,8 @@ dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], - [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) + [$OPENSSL_INCLUDES $LIBHASHLIB_INCL], + [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_LIBS]) dnl test modules PY_STDLIB_MOD([_testcapi], From e0d2e784d49d06164b5ed493cdda3e34d60617e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:05:15 +0200 Subject: [PATCH 03/13] post-merge --- Modules/_hashlib/hashlib_mutex.h | 96 +++++++++++++++++++------------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/Modules/_hashlib/hashlib_mutex.h b/Modules/_hashlib/hashlib_mutex.h index cba7524f0cc668..d6924a2ef61e81 100644 --- a/Modules/_hashlib/hashlib_mutex.h +++ b/Modules/_hashlib/hashlib_mutex.h @@ -5,58 +5,78 @@ #include "pycore_lock.h" // PyMutex /* - * Maximum number of bytes for a message for which the GIL is held - * when performing incremental hashing. + * Message length above which the GIL is to be released + * when performing hashing operations. */ #define HASHLIB_GIL_MINSIZE 2048 /* * Helper code to synchronize access to the hash object when the GIL is - * released around a CPU consuming hashlib operation. All code paths that - * access a mutable part of obj must be enclosed in an ENTER_HASHLIB / - * LEAVE_HASHLIB block or explicitly acquire and release the lock inside - * a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for - * an operation. + * released around a CPU consuming hashlib operation. * - * These only drop the GIL if the lock acquisition itself is likely to - * block. Thus the non-blocking acquire gating the GIL release for a - * blocking lock acquisition. The intent of these macros is to surround - * the assumed always "fast" operations that you aren't releasing the - * GIL around. Otherwise use code similar to what you see in hash - * function update() methods. + * Code accessing a mutable part of the hash object must be enclosed in + * an HASHLIB_{ACQUIRE,RELEASE}_LOCK block or explicitly acquire and release + * the mutex inside a Py_BEGIN_ALLOW_THREADS -- Py_END_ALLOW_THREADS block if + * they wish to release the GIL for an operation. */ -/* Prevent undefined behaviors via multiple threads entering the C API. */ -#define HASHLIB_LOCK_HEAD \ - bool use_mutex; \ +#define HASHLIB_OBJECT_HEAD \ + PyObject_HEAD \ + /* Guard against race conditions during incremental update(). */ \ PyMutex mutex; -#ifdef Py_GIL_DISABLED -#define HASHLIB_INIT_MUTEX(OBJ) \ - do { \ - (OBJ)->mutex = (PyMutex){0}; \ - (OBJ)->use_mutex = true; \ +#define HASHLIB_INIT_MUTEX(OBJ) \ + do { \ + (OBJ)->mutex = (PyMutex){0}; \ } while (0) -#else -#define HASHLIB_INIT_MUTEX(OBJ) \ - do { \ - (OBJ)->mutex = (PyMutex){0}; \ - (OBJ)->use_mutex = false; \ - } while (0) -#endif -#define ENTER_HASHLIB(OBJ) \ - do { \ - if ((OBJ)->use_mutex) { \ - PyMutex_Lock(&(OBJ)->mutex); \ - } \ +#define HASHLIB_ACQUIRE_LOCK(OBJ) PyMutex_Lock(&(OBJ)->mutex) +#define HASHLIB_RELEASE_LOCK(OBJ) PyMutex_Unlock(&(OBJ)->mutex) + +// Macros for executing code while conditionally holding the GIL. +// +// These only drop the GIL if the lock acquisition itself is likely to +// block. Thus the non-blocking acquire gating the GIL release for a +// blocking lock acquisition. The intent of these macros is to surround +// the assumed always "fast" operations that you aren't releasing the +// GIL around. + +/* + * Execute a suite of C statements 'STATEMENTS'. + * + * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. + */ +#define HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(SIZE, STATEMENTS) \ + do { \ + if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + STATEMENTS; \ + Py_END_ALLOW_THREADS \ + } \ + else { \ + STATEMENTS; \ + } \ } while (0) -#define LEAVE_HASHLIB(OBJ) \ - do { \ - if ((OBJ)->use_mutex) { \ - PyMutex_Unlock(&(OBJ)->mutex); \ - } \ +/* + * Lock 'OBJ' and execute a suite of C statements 'STATEMENTS'. + * + * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. + */ +#define HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(OBJ, SIZE, STATEMENTS) \ + do { \ + if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + HASHLIB_ACQUIRE_LOCK(OBJ); \ + STATEMENTS; \ + HASHLIB_RELEASE_LOCK(OBJ); \ + Py_END_ALLOW_THREADS \ + } \ + else { \ + HASHLIB_ACQUIRE_LOCK(OBJ); \ + STATEMENTS; \ + HASHLIB_RELEASE_LOCK(OBJ); \ + } \ } while (0) #endif // !_HASHLIB_HASHLIB_MUTEX_H From a8c492d04986bd44f897dbc7858df1ec91544432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:08:08 +0200 Subject: [PATCH 04/13] post-merge --- PCbuild/pythoncore.vcxproj.filters | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index ad3fc157ff6308..9cc395cac31b15 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -256,13 +256,13 @@ Include - Modules + Modules\_hashlib - Modules + Modules\_hashlib - Modules + Modules\_hashlib Modules @@ -981,10 +981,10 @@ Modules - Modules + Modules\_hashlib - Modules + Modules\_hashlib Modules From 71a1b8f7896fd4e22c6839d52efdd333a9e5e20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:19:11 +0200 Subject: [PATCH 05/13] simplify CI --- .github/workflows/build.yml | 375 ++++++++++++++++++++++++++++++++++-- 1 file changed, 362 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6312530669623e..e09ee877c821c0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,12 @@ jobs: # uses: ./.github/workflows/reusable-context.yml + check-docs: + name: Docs + needs: build-context + if: fromJSON(needs.build-context.outputs.run-docs) + uses: ./.github/workflows/reusable-docs.yml + check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job @@ -154,12 +160,15 @@ jobs: needs: build-context if: fromJSON(needs.build-context.outputs.run-windows-tests) strategy: - fail-fast: true + fail-fast: false matrix: arch: - x64 + - Win32 + - arm64 free-threading: - false + - true exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } @@ -168,6 +177,22 @@ jobs: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} + build-windows-msi: + name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category + Windows MSI${{ '' }} + needs: build-context + if: fromJSON(needs.build-context.outputs.run-windows-msi) + strategy: + fail-fast: false + matrix: + arch: + - x86 + - x64 + - arm64 + uses: ./.github/workflows/reusable-windows-msi.yml + with: + arch: ${{ matrix.arch }} + build-macos: name: >- macOS @@ -175,17 +200,20 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: true + fail-fast: false matrix: # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. # macOS 13 only runs tests against the GIL-enabled CPython. # Cirrus used for upstream, macos-14 for forks. os: - ghcr.io/cirruslabs/macos-runner:sonoma + - macos-14 + - macos-13 is-fork: # only used for the exclusion trick - ${{ github.repository_owner != 'python' }} free-threading: - false + - true exclude: - os: ghcr.io/cirruslabs/macos-runner:sonoma is-fork: true @@ -207,14 +235,17 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: true + fail-fast: false matrix: bolt: - false + - true free-threading: - false + - true os: - ubuntu-24.04 + - ubuntu-24.04-arm exclude: # Do not test BOLT with free-threading, to conserve resources - bolt: true @@ -229,17 +260,17 @@ jobs: free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} - build-ubuntu-ssltests: + build-ubuntu-ssltests-openssl: name: 'Ubuntu SSL tests with OpenSSL' runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: true + fail-fast: false matrix: os: [ubuntu-24.04] - openssl_ver: [3.0.16] + openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} @@ -291,6 +322,81 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build-ubuntu-ssltests-awslc: + name: 'Ubuntu SSL tests with AWS-LC' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04] + awslc_ver: [1.55.0] + env: + AWSLC_VER: ${{ matrix.awslc_ver}} + MULTISSL_DIR: ${{ github.workspace }}/multissl + OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure SSL lib env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore AWS-LC build' + id: cache-aws-lc + uses: actions/cache@v4 + with: + path: ./multissl/aws-lc/${{ matrix.awslc_ver }} + key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} + - name: Install AWS-LC + if: steps.cache-aws-lc.outputs.cache-hit != 'true' + run: | + python3 Tools/ssl/multissltests.py \ + --steps=library \ + --base-directory "$MULTISSL_DIR" \ + --awslc ${{ matrix.awslc_ver }} \ + --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false + - name: Configure CPython + run: | + ./configure CFLAGS="-fdiagnostics-format=json" \ + --config-cache \ + --enable-slower-safety \ + --with-pydebug \ + --with-openssl="$OPENSSL_DIR" \ + --with-builtin-hashlib-hashes=blake2 \ + --with-ssl-default-suites=openssl + - name: Build CPython + run: make -j + - name: Display build info + run: make pythoninfo + - name: Verify python is linked to AWS-LC + run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + - name: SSL tests + run: ./python Lib/test/ssltests.py + build-wasi: name: 'WASI' needs: build-context @@ -299,6 +405,211 @@ jobs: with: config_hash: ${{ needs.build-context.outputs.config-hash }} + test-hypothesis: + name: "Hypothesis tests on Ubuntu" + runs-on: ubuntu-24.04 + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + env: + OPENSSL_VER: 3.0.16 + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" + - name: Create directories for read-only out-of-tree builds + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" + - name: Bind mount sources read-only + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-pydebug \ + --enable-slower-safety \ + --with-openssl="$OPENSSL_DIR" + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j4 + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" + - name: "Create hypothesis venv" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + VENV_LOC=$(realpath -m .)/hypovenv + VENV_PYTHON=$VENV_LOC/bin/python + echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" + echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" + ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" + - name: 'Restore Hypothesis database' + id: cache-hypothesis-database + uses: actions/cache@v4 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ + key: hypothesis-database-${{ github.head_ref || github.run_id }} + restore-keys: | + hypothesis-database- + - name: "Run tests" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + # Most of the excluded tests are slow test suites with no property tests + # + # (GH-104097) test_sysconfig is skipped because it has tests that are + # failing when executed from inside a virtual environment. + "${VENV_PYTHON}" -m test \ + -W \ + --slowest \ + -j4 \ + --timeout 900 \ + -x test_asyncio \ + -x test_multiprocessing_fork \ + -x test_multiprocessing_forkserver \ + -x test_multiprocessing_spawn \ + -x test_concurrent_futures \ + -x test_socket \ + -x test_subprocess \ + -x test_signal \ + -x test_sysconfig + - uses: actions/upload-artifact@v4 + if: always() + with: + name: hypothesis-example-db + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ + + build-asan: + name: 'Address sanitizer' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04] + env: + OPENSSL_VER: 3.0.16 + PYTHONSTRICTEXTENSIONBUILD: 1 + ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Set up GCC-10 for ASAN + uses: egor-tensin/setup-gcc@v1 + with: + version: 10 + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: ./configure --config-cache --with-address-sanitizer --without-pymalloc + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: xvfb-run make ci + + build-tsan: + name: >- + Thread sanitizer + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + free-threading: + - false + - true + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.build-context.outputs.config-hash }} + free-threading: ${{ matrix.free-threading }} + + build-ubsan: + name: Undefined behavior sanitizer + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + uses: ./.github/workflows/reusable-ubsan.yml + with: + config_hash: ${{ needs.build-context.outputs.config-hash }} + cross-build-linux: name: Cross build Linux runs-on: ubuntu-latest @@ -339,6 +650,45 @@ jobs: run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ + cifuzz: + name: CIFuzz + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-ci-fuzz == 'true' + permissions: + security-events: write + strategy: + fail-fast: false + matrix: + sanitizer: [address, undefined, memory] + steps: + - name: Build fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: cpython3 + sanitizer: ${{ matrix.sanitizer }} + - name: Run fuzzers (${{ matrix.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: cpython3 + output-sarif: true + sanitizer: ${{ matrix.sanitizer }} + - name: Upload crash + if: failure() && steps.build.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass @@ -348,12 +698,9 @@ jobs: - build-context # Transitive dependency, needed to access `run-tests` value - check-autoconf-regen - check-generated-files - - build-windows - - build-macos - - build-ubuntu - - build-ubuntu-ssltests + - build-ubuntu-ssltests-awslc + - build-ubuntu-ssltests-openssl - build-wasi - - cross-build-linux if: always() steps: @@ -362,7 +709,8 @@ jobs: with: allowed-failures: >- build-windows-msi, - build-ubuntu-ssltests, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, test-hypothesis, cifuzz, allowed-skips: >- @@ -380,7 +728,8 @@ jobs: check-generated-files, build-macos, build-ubuntu, - build-ubuntu-ssltests, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, build-wasi, test-hypothesis, build-asan, From a60b5cae287ae1a77b2fffe47157c8a465a51969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:21:54 +0200 Subject: [PATCH 06/13] simplify CI --- .github/workflows/build.yml | 542 +----------------------------------- 1 file changed, 1 insertion(+), 541 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e09ee877c821c0..f52f8828505b04 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,12 +43,6 @@ jobs: # uses: ./.github/workflows/reusable-context.yml - check-docs: - name: Docs - needs: build-context - if: fromJSON(needs.build-context.outputs.run-docs) - uses: ./.github/workflows/reusable-docs.yml - check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job @@ -177,226 +171,6 @@ jobs: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} - build-windows-msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} - needs: build-context - if: fromJSON(needs.build-context.outputs.run-windows-msi) - strategy: - fail-fast: false - matrix: - arch: - - x86 - - x64 - - arm64 - uses: ./.github/workflows/reusable-windows-msi.yml - with: - arch: ${{ matrix.arch }} - - build-macos: - name: >- - macOS - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. - # macOS 13 only runs tests against the GIL-enabled CPython. - # Cirrus used for upstream, macos-14 for forks. - os: - - ghcr.io/cirruslabs/macos-runner:sonoma - - macos-14 - - macos-13 - is-fork: # only used for the exclusion trick - - ${{ github.repository_owner != 'python' }} - free-threading: - - false - - true - exclude: - - os: ghcr.io/cirruslabs/macos-runner:sonoma - is-fork: true - - os: macos-14 - is-fork: false - - os: macos-13 - free-threading: true - uses: ./.github/workflows/reusable-macos.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - free-threading: ${{ matrix.free-threading }} - os: ${{ matrix.os }} - - build-ubuntu: - name: >- - Ubuntu - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - bolt: - - false - - true - free-threading: - - false - - true - os: - - ubuntu-24.04 - - ubuntu-24.04-arm - exclude: - # Do not test BOLT with free-threading, to conserve resources - - bolt: true - free-threading: true - # BOLT currently crashes during instrumentation on aarch64 - - os: ubuntu-24.04-arm - bolt: true - uses: ./.github/workflows/reusable-ubuntu.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - bolt-optimizations: ${{ matrix.bolt }} - free-threading: ${{ matrix.free-threading }} - os: ${{ matrix.os }} - - build-ubuntu-ssltests-openssl: - name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] - # See Tools/ssl/make_ssl_data.py for notes on adding a new version - env: - OPENSSL_VER: ${{ matrix.openssl_ver }} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Configure CPython - run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: SSL tests - run: ./python Lib/test/ssltests.py - - build-ubuntu-ssltests-awslc: - name: 'Ubuntu SSL tests with AWS-LC' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - awslc_ver: [1.55.0] - env: - AWSLC_VER: ${{ matrix.awslc_ver}} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure SSL lib env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore AWS-LC build' - id: cache-aws-lc - uses: actions/cache@v4 - with: - path: ./multissl/aws-lc/${{ matrix.awslc_ver }} - key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} - - name: Install AWS-LC - if: steps.cache-aws-lc.outputs.cache-hit != 'true' - run: | - python3 Tools/ssl/multissltests.py \ - --steps=library \ - --base-directory "$MULTISSL_DIR" \ - --awslc ${{ matrix.awslc_ver }} \ - --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Configure CPython - run: | - ./configure CFLAGS="-fdiagnostics-format=json" \ - --config-cache \ - --enable-slower-safety \ - --with-pydebug \ - --with-openssl="$OPENSSL_DIR" \ - --with-builtin-hashlib-hashes=blake2 \ - --with-ssl-default-suites=openssl - - name: Build CPython - run: make -j - - name: Display build info - run: make pythoninfo - - name: Verify python is linked to AWS-LC - run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC - - name: SSL tests - run: ./python Lib/test/ssltests.py - build-wasi: name: 'WASI' needs: build-context @@ -405,291 +179,6 @@ jobs: with: config_hash: ${{ needs.build-context.outputs.config-hash }} - test-hypothesis: - name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-24.04 - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - env: - OPENSSL_VER: 3.0.16 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" - echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - - name: Create directories for read-only out-of-tree builds - run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - - name: Bind mount sources read-only - run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --enable-slower-safety \ - --with-openssl="$OPENSSL_DIR" - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - - name: "Create hypothesis venv" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - VENV_LOC=$(realpath -m .)/hypovenv - VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" - echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" - ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - - name: 'Restore Hypothesis database' - id: cache-hypothesis-database - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ - key: hypothesis-database-${{ github.head_ref || github.run_id }} - restore-keys: | - hypothesis-database- - - name: "Run tests" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - # Most of the excluded tests are slow test suites with no property tests - # - # (GH-104097) test_sysconfig is skipped because it has tests that are - # failing when executed from inside a virtual environment. - "${VENV_PYTHON}" -m test \ - -W \ - --slowest \ - -j4 \ - --timeout 900 \ - -x test_asyncio \ - -x test_multiprocessing_fork \ - -x test_multiprocessing_forkserver \ - -x test_multiprocessing_spawn \ - -x test_concurrent_futures \ - -x test_socket \ - -x test_subprocess \ - -x test_signal \ - -x test_sysconfig - - uses: actions/upload-artifact@v4 - if: always() - with: - name: hypothesis-example-db - path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ - - build-asan: - name: 'Address sanitizer' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - env: - OPENSSL_VER: 3.0.16 - PYTHONSTRICTEXTENSIONBUILD: 1 - ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 10 - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: xvfb-run make ci - - build-tsan: - name: >- - Thread sanitizer - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - free-threading: - - false - - true - uses: ./.github/workflows/reusable-tsan.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - free-threading: ${{ matrix.free-threading }} - - build-ubsan: - name: Undefined behavior sanitizer - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - uses: ./.github/workflows/reusable-ubsan.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - - cross-build-linux: - name: Cross build Linux - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Set build dir - run: - # an absolute path outside of the working directoy - echo "BUILD_DIR=$(realpath ${{ github.workspace }}/../build)" >> "$GITHUB_ENV" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure host build - run: ./configure --prefix="$BUILD_DIR/host-python" - - name: Install host Python - run: make -j8 install - - name: Run test subset with host build - run: | - "$BUILD_DIR/host-python/bin/python3" -m test test_sysconfig test_site test_embed - - name: Configure cross build - run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" - - name: Install cross Python - run: make -j8 install - - name: Run test subset with host build - run: | - "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed - - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ - cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-ci-fuzz == 'true' - permissions: - security-events: write - strategy: - fail-fast: false - matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 - oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif - all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass runs-on: ubuntu-latest @@ -698,8 +187,7 @@ jobs: - build-context # Transitive dependency, needed to access `run-tests` value - check-autoconf-regen - check-generated-files - - build-ubuntu-ssltests-awslc - - build-ubuntu-ssltests-openssl + - build-windows - build-wasi if: always() @@ -707,34 +195,13 @@ jobs: - name: Check whether the needed jobs succeeded or failed uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: - allowed-failures: >- - build-windows-msi, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, - test-hypothesis, - cifuzz, allowed-skips: >- - ${{ - !fromJSON(needs.build-context.outputs.run-docs) - && ' - check-docs, - ' - || '' - }} ${{ needs.build-context.outputs.run-tests != 'true' && ' check-autoconf-regen, check-generated-files, - build-macos, - build-ubuntu, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, build-wasi, - test-hypothesis, - build-asan, - build-tsan, - cross-build-linux, ' || '' }} @@ -745,11 +212,4 @@ jobs: ' || '' }} - ${{ - !fromJSON(needs.build-context.outputs.run-ci-fuzz) - && ' - cifuzz, - ' - || '' - }} jobs: ${{ toJSON(needs) }} From 0845dcf878b8815b399fa2abb08343e32b926cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:25:00 +0200 Subject: [PATCH 07/13] less runs --- .github/workflows/build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f52f8828505b04..891b83403f85f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -158,11 +158,8 @@ jobs: matrix: arch: - x64 - - Win32 - - arm64 free-threading: - false - - true exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } From 7a45a9be467159efff5aba67574305be478881cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:50:05 +0200 Subject: [PATCH 08/13] post-merge --- Modules/_hashlib/hashlib_fetch.h | 73 ++++++++++++++++++-------------- Modules/_hashopenssl.c | 6 ++- Modules/hmacmodule.c | 3 +- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h index ee647ea6220f4d..833016d1267fdc 100644 --- a/Modules/_hashlib/hashlib_fetch.h +++ b/Modules/_hashlib/hashlib_fetch.h @@ -50,56 +50,65 @@ #include "Python.h" -#define Py_HASHLIB_MD_NS(ATTR) Py_hashlib_message_digest_ ## ATTR -#define Py_HASHLIB_MD_FAMILY(FAMILY_ID) Py_HASHLIB_MD_NS(family_ ## FAMILY_ID) -#define Py_HASHLIB_MD_MEMBER(MEMBER_ID) Py_HASHLIB_MD_NS(member_ ## MEMBER_ID) +/* + * Internal error messages used for reporting an unsupported hash algorithm. + * The algorithm can be given by its name, a callable or a PEP-247 module. + * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() + * and _hmacmodule.c::find_hash_info(). + */ +#define _Py_HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" +#define _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" + +#define _Py_HASHLIB_MD_NS(ATTR) _Py_hashlib_message_digest_ ## ATTR +#define _Py_HASHLIB_MD_FAMILY(FAMILY) _Py_HASHLIB_MD_NS(family_ ## FAMILY) +#define _Py_HASHLIB_MD_MEMBER(MEMBER) _Py_HASHLIB_MD_NS(member_ ## MEMBER) -#define Py_HASHLIB_MD_NAMES Py_HASHLIB_MD_NS(NAMES) -#define Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(Py_HASHLIB_MD_NAMES) -#define Py_HASHLIB_MD_NAME(MEMBER_ID) \ +#define _Py_HASHLIB_MD_NAMES _Py_HASHLIB_MD_NS(NAMES) +#define _Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(Py_HASHLIB_MD_NAMES) +#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ ( \ assert(Py_HASHLIB_MD_NAME(MEMBER_ID) < Py_HASHLIB_MD_COUNT), \ Py_HASHLIB_MD_NAMES[Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ ) typedef enum { - Py_HASHLIB_MD_FAMILY(MD) = 0, - Py_HASHLIB_MD_FAMILY(SHA1), - Py_HASHLIB_MD_FAMILY(SHA2), - Py_HASHLIB_MD_FAMILY(SHA2t), - Py_HASHLIB_MD_FAMILY(SHA3), - Py_HASHLIB_MD_FAMILY(SHA3_XOF), - Py_HASHLIB_MD_FAMILY(BLAKE2), -} Py_HASHLIB_MD_NS(family); + _Py_HASHLIB_MD_FAMILY(MD) = 0, + _Py_HASHLIB_MD_FAMILY(SHA1), + _Py_HASHLIB_MD_FAMILY(SHA2), + _Py_HASHLIB_MD_FAMILY(SHA2t), + _Py_HASHLIB_MD_FAMILY(SHA3), + _Py_HASHLIB_MD_FAMILY(SHA3_XOF), + _Py_HASHLIB_MD_FAMILY(BLAKE2), +} _Py_HASHLIB_MD_NS(family); typedef enum { /* MD-family */ - Py_HASHLIB_MD_MEMBER(md5) = 0, + _Py_HASHLIB_MD_MEMBER(md5) = 0, /* SHA-1 family */ - Py_HASHLIB_MD_MEMBER(sha1), + _Py_HASHLIB_MD_MEMBER(sha1), /* SHA-2 family */ - Py_HASHLIB_MD_MEMBER(sha224), - Py_HASHLIB_MD_MEMBER(sha256), - Py_HASHLIB_MD_MEMBER(sha384), - Py_HASHLIB_MD_MEMBER(sha512), + _Py_HASHLIB_MD_MEMBER(sha224), + _Py_HASHLIB_MD_MEMBER(sha256), + _Py_HASHLIB_MD_MEMBER(sha384), + _Py_HASHLIB_MD_MEMBER(sha512), /* Truncated SHA-2 family */ - Py_HASHLIB_MD_MEMBER(sha512_224), - Py_HASHLIB_MD_MEMBER(sha512_256), + _Py_HASHLIB_MD_MEMBER(sha512_224), + _Py_HASHLIB_MD_MEMBER(sha512_256), /* SHA-3 family */ - Py_HASHLIB_MD_MEMBER(sha3_224), - Py_HASHLIB_MD_MEMBER(sha3_256), - Py_HASHLIB_MD_MEMBER(sha3_384), - Py_HASHLIB_MD_MEMBER(sha3_512), + _Py_HASHLIB_MD_MEMBER(sha3_224), + _Py_HASHLIB_MD_MEMBER(sha3_256), + _Py_HASHLIB_MD_MEMBER(sha3_384), + _Py_HASHLIB_MD_MEMBER(sha3_512), /* SHA-3 XOF SHAKE family */ - Py_HASHLIB_MD_MEMBER(shake_128), - Py_HASHLIB_MD_MEMBER(shake_256), + _Py_HASHLIB_MD_MEMBER(shake_128), + _Py_HASHLIB_MD_MEMBER(shake_256), /* BLAKE-2 family */ - Py_HASHLIB_MD_MEMBER(blake2b), - Py_HASHLIB_MD_MEMBER(blake2s), -} Py_HASHLIB_MD_NS(member); + _Py_HASHLIB_MD_MEMBER(blake2b), + _Py_HASHLIB_MD_MEMBER(blake2s), +} _Py_HASHLIB_MD_NS(member); static const char *Py_HASHLIB_MD_NAMES[] = { -#define DECL_MESSAGE_DIGEST_NAME(ID) [Py_HASHLIB_MD_MEMBER(ID)] = #ID +#define DECL_MESSAGE_DIGEST_NAME(ID) [_Py_HASHLIB_MD_MEMBER(ID)] = #ID /* MD-family */ DECL_MESSAGE_DIGEST_NAME(md5), /* SHA-1 family */ diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 0d98063dfc2108..c42513a53d4561 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -26,7 +26,9 @@ #include "pycore_hashtable.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED + #include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_fetch.h" #include "_hashlib/hashlib_mutex.h" /* EVP is the preferred interface to hashing in OpenSSL */ @@ -533,7 +535,7 @@ raise_unsupported_algorithm_error(_hashlibstate *state, PyObject *digestmod) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - HASHLIB_UNSUPPORTED_ALGORITHM, + _Py_HASHLIB_UNSUPPORTED_ALGORITHM, digestmod ); } @@ -543,7 +545,7 @@ raise_unsupported_str_algorithm_error(_hashlibstate *state, const char *name) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - HASHLIB_UNSUPPORTED_STR_ALGORITHM, + _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM, name ); } diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index a058d33d0ca812..4f132f38eb6f2f 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -46,6 +46,7 @@ #include "_hacl/Hacl_Streaming_Types.h" // Hacl_Streaming_Types_error_code #include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_fetch.h" #include "_hashlib/hashlib_mutex.h" #include @@ -657,7 +658,7 @@ find_hash_info(hmacmodule_state *state, PyObject *hash_info_ref) } if (rc == 0) { PyErr_Format(state->unknown_hash_error, - HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); + _Py_HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); return NULL; } assert(info != NULL); From cbaaa40a64f33e627cd350d8cc3ec77c1732ae9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:52:07 +0200 Subject: [PATCH 09/13] post-merge --- Modules/_hashlib/hashlib_fetch.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h index 833016d1267fdc..2cfabc6b25aac1 100644 --- a/Modules/_hashlib/hashlib_fetch.h +++ b/Modules/_hashlib/hashlib_fetch.h @@ -59,16 +59,16 @@ #define _Py_HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" #define _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" -#define _Py_HASHLIB_MD_NS(ATTR) _Py_hashlib_message_digest_ ## ATTR -#define _Py_HASHLIB_MD_FAMILY(FAMILY) _Py_HASHLIB_MD_NS(family_ ## FAMILY) -#define _Py_HASHLIB_MD_MEMBER(MEMBER) _Py_HASHLIB_MD_NS(member_ ## MEMBER) +#define _Py_HASHLIB_MD_NAMESPACE(NAME) _Py_hashlib_message_digest_ ## NAME +#define _Py_HASHLIB_MD_FAMILY(ID) _Py_HASHLIB_MD_NAMESPACE(family_ ## ID) +#define _Py_HASHLIB_MD_MEMBER(ID) _Py_HASHLIB_MD_NAMESPACE(member_ ## ID) -#define _Py_HASHLIB_MD_NAMES _Py_HASHLIB_MD_NS(NAMES) -#define _Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(Py_HASHLIB_MD_NAMES) -#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ +#define _Py_HASHLIB_MD_NAMES _Py_HASHLIB_MD_NAMESPACE(NAMES) +#define _Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(_Py_HASHLIB_MD_NAMES) +#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ ( \ - assert(Py_HASHLIB_MD_NAME(MEMBER_ID) < Py_HASHLIB_MD_COUNT), \ - Py_HASHLIB_MD_NAMES[Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ + assert(_Py_HASHLIB_MD_NAME(MEMBER_ID) < _Py_HASHLIB_MD_COUNT), \ + _Py_HASHLIB_MD_NAMES[_Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ ) typedef enum { @@ -79,7 +79,7 @@ typedef enum { _Py_HASHLIB_MD_FAMILY(SHA3), _Py_HASHLIB_MD_FAMILY(SHA3_XOF), _Py_HASHLIB_MD_FAMILY(BLAKE2), -} _Py_HASHLIB_MD_NS(family); +} _Py_HASHLIB_MD_NAMESPACE(family); typedef enum { /* MD-family */ @@ -105,7 +105,7 @@ typedef enum { /* BLAKE-2 family */ _Py_HASHLIB_MD_MEMBER(blake2b), _Py_HASHLIB_MD_MEMBER(blake2s), -} _Py_HASHLIB_MD_NS(member); +} _Py_HASHLIB_MD_NAMESPACE(member); static const char *Py_HASHLIB_MD_NAMES[] = { #define DECL_MESSAGE_DIGEST_NAME(ID) [_Py_HASHLIB_MD_MEMBER(ID)] = #ID From c653c2908d664fc2124c43582eddbb3a2fab1726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:02:33 +0200 Subject: [PATCH 10/13] fix --- Makefile.pre.in | 35 +++++++++++++++++++++--------- Modules/_hashlib/hashlib_fetch.h | 11 +++++----- Tools/wasm/wasi/__main__.py | 2 +- configure | 37 ++++++++++++++++++-------------- configure.ac | 15 +++++++------ 5 files changed, 62 insertions(+), 38 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 654d6becc984e7..28501ff1676e73 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -227,6 +227,7 @@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a +LIBHASHLIB_INTERNAL_A=Modules/_hashlib/libhashlib.a # HACL* build configuration LIBHACL_CFLAGS=@LIBHACL_CFLAGS@ @@ -761,6 +762,18 @@ LIBHACL_HMAC_HEADERS= \ $(LIBHACL_BLAKE2_HEADERS) \ $(LIBHACL_HEADERS) +########################################################################## +# Internal library for cryptographic primitives + +LIBHASHLIB_INTERNAL_OBJS= \ + Modules/_hashlib/hashlib_buffer.o \ + Modules/_hashlib/hashlib_fetch.o + +LIBHASHLIB_INTERNAL_HEADERS= \ + Modules/_hashlib/hashlib_buffer.h \ + Modules/_hashlib/hashlib_fetch.h \ + Modules/_hashlib/hashlib_mutex.h + ######################################################################### # Rules @@ -1483,9 +1496,17 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) ########################################################################## # '_hashlib', '_hmac' and HACL*-based modules helpers +LIBHASHLIB_INTERNAL_CFLAGS=@LIBHASHLIB_INTERNAL_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) -Modules/_hashlib/hashlib_buffer.o: $(srcdir)/Modules/_hashlib/hashlib_buffer.c $(srcdir)/Modules/_hashlib/hashlib_buffer.h $(PYTHON_HEADERS) - $(CC) -I$(srcdir)/Modules/_hashlib -c $(PY_STDMODULE_CFLAGS) $(CCSHARED) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c +Modules/_hashlib/hashlib_buffer.o: Modules/_hashlib/hashlib_buffer.c $(LIBHASHLIB_INTERNAL_HEADERS) $(PYTHON_HEADERS) + $(CC) -I$(srcdir)/Modules/_hashlib -c $(LIBHASHLIB_INTERNAL_CFLAGS) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c + +Modules/_hashlib/hashlib_fetch.o: Modules/_hashlib/hashlib_fetch.c $(LIBHASHLIB_INTERNAL_HEADERS) $(PYTHON_HEADERS) + $(CC) -I$(srcdir)/Modules/_hashlib -c $(LIBHASHLIB_INTERNAL_CFLAGS) -o $@ $(srcdir)/Modules/_hashlib/hashlib_fetch.c + +$(LIBHASHLIB_INTERNAL_A): $(LIBHASHLIB_INTERNAL_OBJS) + -rm -f $@ + $(AR) $(ARFLAGS) $@ $(LIBHASHLIB_INTERNAL_OBJS) ########################################################################## # HACL* library build @@ -3331,14 +3352,8 @@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h -MODULE__HASHLIB_DEPS= \ - $(srcdir)/Modules/_hashlib/hashlib_buffer.h \ - $(srcdir)/Modules/_hashlib/hashlib_fetch.h \ - $(srcdir)/Modules/_hashlib/hashlib_mutex.h - -MODULE__HASHLIB_LDEPS= \ - Modules/_hashlib/hashlib_buffer.o \ - Modules/_hashlib/hashlib_fetch.o +MODULE__HASHLIB_DEPS=@LIBHASHLIB_INTERNAL@ +MODULE__HASHLIB_LDEPS= # HACL*-based cryptographic primitives MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h index 2cfabc6b25aac1..a583b737923e21 100644 --- a/Modules/_hashlib/hashlib_fetch.h +++ b/Modules/_hashlib/hashlib_fetch.h @@ -65,10 +65,11 @@ #define _Py_HASHLIB_MD_NAMES _Py_HASHLIB_MD_NAMESPACE(NAMES) #define _Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(_Py_HASHLIB_MD_NAMES) -#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ - ( \ - assert(_Py_HASHLIB_MD_NAME(MEMBER_ID) < _Py_HASHLIB_MD_COUNT), \ - _Py_HASHLIB_MD_NAMES[_Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ +#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ + ( \ + assert(_Py_HASHLIB_MD_MEMBER(MEMBER_ID) >= 0), \ + assert(_Py_HASHLIB_MD_MEMBER(MEMBER_ID) < _Py_HASHLIB_MD_COUNT), \ + _Py_HASHLIB_MD_NAMES[_Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ ) typedef enum { @@ -107,7 +108,7 @@ typedef enum { _Py_HASHLIB_MD_MEMBER(blake2s), } _Py_HASHLIB_MD_NAMESPACE(member); -static const char *Py_HASHLIB_MD_NAMES[] = { +static const char *_Py_HASHLIB_MD_NAMES[] = { #define DECL_MESSAGE_DIGEST_NAME(ID) [_Py_HASHLIB_MD_MEMBER(ID)] = #ID /* MD-family */ DECL_MESSAGE_DIGEST_NAME(md5), diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index 54ccc95157d57d..fed874013e87cd 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -178,7 +178,7 @@ def find_wasi_sdk(): def wasi_sdk_env(context): """Calculate environment variables for building with wasi-sdk.""" - wasi_sdk_path = context.wasi_sdk_path + wasi_sdk_path = context.wasi_sdk_path.absolute() sysroot = wasi_sdk_path / "share" / "wasi-sysroot" env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", "AR": "llvm-ar", "RANLIB": "ranlib"} diff --git a/configure b/configure index f76aaead2b57fc..cb5f9e4ec5c0fd 100755 --- a/configure +++ b/configure @@ -725,6 +725,8 @@ LIBHACL_BLAKE2_SIMD128_OBJS LIBHACL_SIMD128_FLAGS LIBHACL_LDFLAGS LIBHACL_CFLAGS +LIBHASHLIB_INTERNAL +LIBHASHLIB_INTERNAL_CFLAGS MODULE_UNICODEDATA_FALSE MODULE_UNICODEDATA_TRUE MODULE__MULTIBYTECODEC_FALSE @@ -32528,8 +32530,11 @@ fi ############################################################################### # Cryptographic primitives -LIBHASHLIB_INCL='-I$(srcdir)/Modules/_hashlib' -LIBHASHLIB_LIBS='Modules/_hashlib/hashlib_buffer.o Modules/_hashlib/hashlib_fetch.o' +LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" +LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" +LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" + + ############################################################################### @@ -32780,8 +32785,8 @@ fi if test "x$py_cv_module__md5" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__md5" = yes; then @@ -32825,8 +32830,8 @@ fi if test "x$py_cv_module__sha1" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha1" = yes; then @@ -32870,8 +32875,8 @@ fi if test "x$py_cv_module__sha2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha2" = yes; then @@ -32915,8 +32920,8 @@ fi if test "x$py_cv_module__sha3" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha3" = yes; then @@ -32960,8 +32965,8 @@ fi if test "x$py_cv_module__blake2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__blake2" = yes; then @@ -33006,8 +33011,8 @@ fi if test "x$py_cv_module__hmac" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__hmac" = yes; then @@ -33688,8 +33693,8 @@ fi if test "x$py_cv_module__hashlib" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__hashlib" = yes; then diff --git a/configure.ac b/configure.ac index a767d4f62bea7e..3ca67bcf188f3b 100644 --- a/configure.ac +++ b/configure.ac @@ -7961,9 +7961,12 @@ PY_STDLIB_MOD_SIMPLE([unicodedata]) ############################################################################### # Cryptographic primitives -LIBHASHLIB_INCL='-I$(srcdir)/Modules/_hashlib' -LIBHASHLIB_LIBS='Modules/_hashlib/hashlib_buffer.o Modules/_hashlib/hashlib_fetch.o' +LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" +LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" +LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" +AC_SUBST([LIBHASHLIB_INTERNAL_CFLAGS]) +AC_SUBST([LIBHASHLIB_INTERNAL]) ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) @@ -8102,8 +8105,8 @@ AC_DEFUN([PY_HACL_CREATE_MODULE], [ AS_VAR_PUSHDEF([v], [[LIBHACL_][$1][_LDFLAGS]]) AS_VAR_SET([v], [[LIBHACL_][$1][_LIB_${LIBHACL_LDEPS_LIBTYPE}]]) PY_STDLIB_MOD([$2], [$3], [], - [$LIBHACL_CFLAGS $LIBHASHLIB_INCL], - [\$($v) $LIBHASHLIB_LIBS]) + [$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS], + [\$($v) $LIBHASHLIB_INTERNAL_LDFLAGS]) AS_VAR_POPDEF([v]) ]) @@ -8184,8 +8187,8 @@ dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], - [$OPENSSL_INCLUDES $LIBHASHLIB_INCL], - [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_LIBS]) + [$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS], + [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS]) dnl test modules PY_STDLIB_MOD([_testcapi], From 826af90e079289ce32fd004d9158e8cd98faf627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:10:55 +0200 Subject: [PATCH 11/13] fixup --- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 4716b8bf11fcbf..5cd494dfd66722 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -238,7 +238,7 @@ Modules/_decimal/_decimal.c - signal_map_template - Modules/_decimal/_decimal.c - ssize_constants - Modules/_decimal/_decimal.c - INVALID_SIGNALDICT_ERROR_MSG - Modules/_elementtree.c - ExpatMemoryHandler - -Modules/_hashlib/hashlib_fetch.h - Py_hashlib_message_digest_NAMES - +Modules/_hashlib/hashlib_fetch.h - _Py_hashlib_message_digest_NAMES - Modules/_hashopenssl.c - py_hashes - Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - Modules/_hacl/Hacl_Hash_MD5.c - _h0 - From 2efb676feb94ac81f67417059a8f7d31b1296737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:26:40 +0200 Subject: [PATCH 12/13] fix --- Makefile.pre.in | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 28501ff1676e73..7d149a52a646fe 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3350,25 +3350,22 @@ MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ -MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h - MODULE__HASHLIB_DEPS=@LIBHASHLIB_INTERNAL@ -MODULE__HASHLIB_LDEPS= +MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h # HACL*-based cryptographic primitives MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__MD5_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA1_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA1_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA2_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA3_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA3_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__BLAKE2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__BLAKE2_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) - +MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__HMAC_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__HMAC_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h From 972a185a9df76c868ad648de851d82a63c0a5f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:39:35 +0200 Subject: [PATCH 13/13] fixup --- PCbuild/pythoncore.vcxproj | 1 + Tools/wasm/wasi/__main__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 7fbb8f9dc7609c..a83ede77fb7f39 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -449,6 +449,7 @@ + diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index fed874013e87cd..54ccc95157d57d 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -178,7 +178,7 @@ def find_wasi_sdk(): def wasi_sdk_env(context): """Calculate environment variables for building with wasi-sdk.""" - wasi_sdk_path = context.wasi_sdk_path.absolute() + wasi_sdk_path = context.wasi_sdk_path sysroot = wasi_sdk_path / "share" / "wasi-sysroot" env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", "AR": "llvm-ar", "RANLIB": "ranlib"} pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy