From 68989a34eab7143ec8ea317621d1fb4b1156688f Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Fri, 14 Mar 2025 19:59:49 +0100 Subject: [PATCH 01/12] Add a dummy non-empty environment when using assert_python_{failure,ok} If the env_vars argument to assert_python_failure or assert_python_ok is empty the test will run in isolated mode (-I) which means that the PYTHONPATH environment variable will be ignored and the test fails because the isal module can not be found, or the test is run usung the system installed version of the module instead of the newly built module that should be tested. By adding a dummy entry to the env_vars argument the isolated mode is not used and the PYTHONPATH environment variable is not ignored and the test works as expected. --- CHANGELOG.rst | 4 ++++ tests/test_gzip_compliance.py | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 805a953a..59757cb1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,10 @@ Changelog .. that users understand how the changes affect the new version. +version 1.7.3 +----------------- ++ Fix an issue where some tests failed because they ignored PYTHONPATH. + version 1.7.2 ----------------- + Use upstream ISA-L version 2.31.1 which includes patches to make diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index cc979317..bd11451d 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -845,9 +845,23 @@ def test_decompress_infile_outfile(self): self.assertTrue(os.path.exists(igzipname)) + # The following tests use assert_python_failure or assert_python_ok. + # + # If the env_vars argument to assert_python_failure or assert_python_ok + # is empty the test will run in isolated mode (-I) which means that the + # PYTHONPATH environment variable will be ignored and the test fails + # because the isal module can not be found, or the test is run usung the + # system installed version of the module instead of the newly built + # module that should be tested. + # + # By adding a dummy entry to the env_vars argument the isolated mode is + # not used and the PYTHONPATH environment variable is not ignored and + # the test works as expected. + def test_decompress_infile_outfile_error(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '-d', - 'thisisatest.out') + 'thisisatest.out', + **{'_dummy': '1'}) self.assertEqual(b"filename doesn't end in .gz: 'thisisatest.out'. " b"Cannot determine output filename.", err.strip()) @@ -872,7 +886,8 @@ def test_compress_infile_outfile_default(self): with open(local_testigzip, 'wb') as fp: fp.write(self.data) - rc, out, err = assert_python_ok('-m', 'isal.igzip', local_testigzip) + rc, out, err = assert_python_ok('-m', 'isal.igzip', local_testigzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(igzipname)) self.assertEqual(out, b'') @@ -891,7 +906,8 @@ def test_compress_infile_outfile(self): rc, out, err = assert_python_ok('-m', 'isal.igzip', compress_level, - local_testigzip) + local_testigzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(igzipname)) self.assertEqual(out, b'') @@ -901,7 +917,7 @@ def test_compress_infile_outfile(self): def test_compress_fast_best_are_exclusive(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '--fast', - '--best') + '--best', **{'_dummy': '1'}) self.assertIn( b"error: argument -3/--best: not allowed with argument -0/--fast", err) @@ -909,7 +925,7 @@ def test_compress_fast_best_are_exclusive(self): def test_decompress_cannot_have_flags_compression(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '--fast', - '-d') + '-d', **{'_dummy': '1'}) self.assertIn( b'error: argument -d/--decompress: not allowed with argument ' b'-0/--fast', From d08488ba37e068703ddff6168bc578d2dcdf8463 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 18 Mar 2025 11:32:50 +0100 Subject: [PATCH 02/12] Update changelog version --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 59757cb1..f9a053b7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,7 +8,7 @@ Changelog .. that users understand how the changes affect the new version. -version 1.7.3 +version 1.8.0-dev ----------------- + Fix an issue where some tests failed because they ignored PYTHONPATH. From 0215519b557f574352620a06255921d8025ecef1 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:08:28 +0200 Subject: [PATCH 03/12] Move to setuptools-scm --- MANIFEST.in | 2 ++ pyproject.toml | 62 +++++++++++++++++++++++++++++++++++++++++++++----- setup.cfg | 2 -- setup.py | 44 +---------------------------------- 4 files changed, 59 insertions(+), 51 deletions(-) delete mode 100644 setup.cfg diff --git a/MANIFEST.in b/MANIFEST.in index 8cc2b029..23ea5299 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,8 @@ graft src/isal/isa-l include src/isal/*.h prune tests prune docs +prune .github +exclude .git* prune benchmark_scripts exclude requirements-docs.txt exclude codecov.yml diff --git a/pyproject.toml b/pyproject.toml index bf5fe99c..425f0f9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,60 @@ [build-system] -requires = ["setuptools>=64", "versioningit>=1.1.0"] +requires = ["setuptools>=77", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" -[tool.versioningit.vcs] -method="git" -default-tag = "v0.0.0" +[project] +name = "isal" +dynamic = ["version"] +description = """ +Faster zlib and gzip compatible compression and decompression by providing \ +python bindings for the ISA-L ibrary.""" +license="PSF-2.0" +keywords=["isal", "isa-l", "compression", "deflate", "gzip", "igzip"] +authors = [{name = "Leiden University Medical Center"}, + {email = "r.h.p.vorderman@lumc.nl"}] +readme = "README.rst" +requires-python = ">=3.8" # BadGzipFile imported +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: C", + "Development Status :: 5 - Production/Stable", + "Topic :: System :: Archiving :: Compression", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", +] +urls.homepage = "https://github.com/pycompression/python-isal" +urls.documentation = "python-isal.readthedocs.io" -[tool.versioningit.write] -file = "src/isal/_version.py" +[tool.setuptools_scm] +version_file = "src/isal/_version.py" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["isal"] + +[tool.setuptools.package-data] +isal = ['*.pyi', 'py.typed', 'isa-l/LICENSE', 'isa-l/README.md', 'isa-l/Release_notes.txt'] +[tool.setuptools.exclude-package-data] +isal = [ + "*.c", + "*.h", + "isa-l/*/*", + "isa-l/Mak*", + "isa-l/.*", + "isa-l/autogen.sh", + "isa-l/Doxyfile", + "isa-l/CONTRIBUTING.md", + "isa-l/SECURITY.md", + "isa-l/configure.ac", + "isa-l/isa-l.*", + "isa-l/libisal.pc.in", + "isa-l/make.inc", +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 256134e1..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -license_files=LICENSE \ No newline at end of file diff --git a/setup.py b/setup.py index 55c89496..1ed64c2d 100644 --- a/setup.py +++ b/setup.py @@ -15,11 +15,9 @@ import tempfile from pathlib import Path -from setuptools import Extension, find_packages, setup +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext -import versioningit - ISA_L_SOURCE = os.path.join("src", "isal", "isa-l") SYSTEM_IS_BSD = (sys.platform.startswith("freebsd") or @@ -138,46 +136,6 @@ def build_isa_l(): setup( - name="isal", - version=versioningit.get_version(), - description="Faster zlib and gzip compatible compression and " - "decompression by providing python bindings for the ISA-L " - "library.", - author="Leiden University Medical Center", - author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now - long_description=Path("README.rst").read_text(), - long_description_content_type="text/x-rst", cmdclass={"build_ext": BuildIsalExt}, - license="PSF-2.0", - keywords="isal isa-l compression deflate gzip igzip threads", - zip_safe=False, - packages=find_packages('src'), - package_dir={'': 'src'}, - package_data={'isal': ['*.pyi', 'py.typed', - # Include isa-l LICENSE and other relevant files - # with the binary distribution. - 'isa-l/LICENSE', 'isa-l/README.md', - 'isa-l/Release_notes.txt']}, - url="https://github.com/pycompression/python-isal", - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: C", - "Development Status :: 5 - Production/Stable", - "Topic :: System :: Archiving :: Compression", - "License :: OSI Approved :: Python Software Foundation License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - ], - python_requires=">=3.8", # BadGzipFile imported ext_modules=EXTENSIONS ) From 1da70a5bb2c4c3e7853064afe91b79098428bc35 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:08:44 +0200 Subject: [PATCH 04/12] Include tests in source distribution --- MANIFEST.in | 1 - 1 file changed, 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 23ea5299..4a48e873 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,5 @@ graft src/isal/isa-l include src/isal/*.h -prune tests prune docs prune .github exclude .git* From 26c2b45de6b9e9d540e924e8b8acbbb4fcd847b7 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:13:24 +0200 Subject: [PATCH 05/12] Fix CFLAGS on default behavior for later setuptools versions --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b55ef76..17dd7f83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,14 +243,14 @@ jobs: CIBW_ENVIRONMENT_LINUX: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=/tmp/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" CIBW_ENVIRONMENT_WINDOWS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}\build_cache CIBW_ENVIRONMENT_MACOS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" - name: Build sdist if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}} run: | From 736644860839a957fa6b47e2aea2f2727c7067ea Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:18:59 +0200 Subject: [PATCH 06/12] Update changelog with build changes --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f9a053b7..b79300d2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,10 @@ Changelog version 1.8.0-dev ----------------- ++ Change build backend to setuptools-scm which is more commonly used and + supported. ++ Include test packages in the source distribution, so source distribution + installations can be verified. + Fix an issue where some tests failed because they ignored PYTHONPATH. version 1.7.2 From fc7d5a56c5d8fb072e1b7a549ecaf1c339046622 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 27 May 2025 13:31:35 +0200 Subject: [PATCH 07/12] Update supported python versions --- .github/workflows/ci.yml | 15 +++++++-------- CHANGELOG.rst | 1 + pyproject.toml | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17dd7f83..b886a730 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.9" - name: Install tox run: pip install tox - name: Lint @@ -36,10 +36,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.9" - name: Install isal run: sudo apt-get install libisal-dev - name: Install tox and upgrade setuptools and pip @@ -54,7 +54,6 @@ jobs: strategy: matrix: python-version: - - "3.8" - "3.9" - "3.10" - "3.11" @@ -65,11 +64,11 @@ jobs: os: ["ubuntu-latest"] include: - os: "macos-13" - python-version: "3.8" + python-version: "3.9" - os: "macos-14" python-version: "3.10" - os: "windows-latest" - python-version: "3.8" + python-version: "3.9" steps: - uses: actions/checkout@v4 with: @@ -105,7 +104,7 @@ jobs: strategy: matrix: python_version: - - "3.8" + - "3.9" steps: - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b79300d2..c49acba7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,7 @@ Changelog version 1.8.0-dev ----------------- ++ Python 3.8 is no longer supported. + Change build backend to setuptools-scm which is more commonly used and supported. + Include test packages in the source distribution, so source distribution diff --git a/pyproject.toml b/pyproject.toml index 425f0f9d..8405984e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,14 +13,15 @@ keywords=["isal", "isa-l", "compression", "deflate", "gzip", "igzip"] authors = [{name = "Leiden University Medical Center"}, {email = "r.h.p.vorderman@lumc.nl"}] readme = "README.rst" -requires-python = ">=3.8" # BadGzipFile imported +requires-python = ">=3.9" # Because of setuptools version classifiers = [ "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", From 5ef868bd29559dd1647230622bae3c8224f27738 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 11:40:33 +0200 Subject: [PATCH 08/12] Fix flush implementation --- src/isal/igzip_threaded.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index 7f1c94fc..355b7f72 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -316,7 +316,7 @@ def write(self, b) -> int: self.input_queues[worker_index].put((data, zdict)) return len(data) - def _end_gzip_stream(self): + def flush(self): self._check_closed() # Wait for all data to be compressed for in_q in self.input_queues: @@ -324,22 +324,18 @@ def _end_gzip_stream(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() - # Write an empty deflate block with a lost block marker. + self.raw.flush() + + def close(self) -> None: + if self._closed: + return + self.flush() self.raw.write(isal_zlib.compress(b"", wbits=-15)) trailer = struct.pack(" None: - if self._closed: - return - self._end_gzip_stream() self.stop() if self.exception: self.raw.close() From 43983269f56a911e84fce09b56cfb95765f6334f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 14:22:54 +0200 Subject: [PATCH 09/12] Add a test for correctly flushing data to disk --- tests/test_igzip_threaded.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 41c61bfe..6331182b 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -12,6 +12,7 @@ import subprocess import sys import tempfile +import zlib from pathlib import Path from isal import igzip_threaded @@ -243,15 +244,28 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): @pytest.mark.parametrize("threads", [1, 2]) def test_flush(tmp_path, threads): + empty_block_end = b"\x00\x00\xff\xff" + deflate_last_block = zlib.compress(b"", wbits=-15) test_file = tmp_path / "output.gz" with igzip_threaded.open(test_file, "wb", threads=threads) as f: f.write(b"1") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"1" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + # Cut off gzip header and end data with an explicit last block to + # test if the data was compressed correctly. + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"1" f.write(b"2") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"12" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"12" f.write(b"3") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"123" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"123" assert gzip.decompress(test_file.read_bytes()) == b"123" From 85e9d4853ca9d67d7b206c71428fc597b31549c6 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 14:26:24 +0200 Subject: [PATCH 10/12] Add multithreaded flushing fix to the CHANGELOG --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c49acba7..508abc46 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,10 @@ Changelog version 1.8.0-dev ----------------- + Python 3.8 is no longer supported. ++ Fix an issue where flushing using igzip_threaded caused a gzip end of stream + and started a new gzip stream. In essence creating a concatenated gzip + stream. Now it is in concordance with how single threaded gzip streams + are flushed using Z_SYNC_FLUSH. + Change build backend to setuptools-scm which is more commonly used and supported. + Include test packages in the source distribution, so source distribution From 0fef0a9bd44689a51d0e32fd51f07708e96e3012 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 24 Jun 2025 14:31:31 +0200 Subject: [PATCH 11/12] Fix test for older versions of python --- tests/test_igzip_threaded.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 6331182b..d2aee567 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -245,7 +245,8 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): @pytest.mark.parametrize("threads", [1, 2]) def test_flush(tmp_path, threads): empty_block_end = b"\x00\x00\xff\xff" - deflate_last_block = zlib.compress(b"", wbits=-15) + compressobj = zlib.compressobj(wbits=-15) + deflate_last_block = compressobj.compress(b"") + compressobj.flush() test_file = tmp_path / "output.gz" with igzip_threaded.open(test_file, "wb", threads=threads) as f: f.write(b"1") From eb3e5c23bbea2e1d91e0d2cb50114c2522870d7f Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Wed, 2 Jul 2025 09:49:25 +0200 Subject: [PATCH 12/12] Fix a typo in a comment --- tests/test_gzip_compliance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index bd11451d..5d71577c 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -850,7 +850,7 @@ def test_decompress_infile_outfile(self): # If the env_vars argument to assert_python_failure or assert_python_ok # is empty the test will run in isolated mode (-I) which means that the # PYTHONPATH environment variable will be ignored and the test fails - # because the isal module can not be found, or the test is run usung the + # because the isal module can not be found, or the test is run using the # system installed version of the module instead of the newly built # module that should be tested. # pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy