diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b55ef76..b886a730 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.9" - name: Install tox run: pip install tox - name: Lint @@ -36,10 +36,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: "3.9" - name: Install isal run: sudo apt-get install libisal-dev - name: Install tox and upgrade setuptools and pip @@ -54,7 +54,6 @@ jobs: strategy: matrix: python-version: - - "3.8" - "3.9" - "3.10" - "3.11" @@ -65,11 +64,11 @@ jobs: os: ["ubuntu-latest"] include: - os: "macos-13" - python-version: "3.8" + python-version: "3.9" - os: "macos-14" python-version: "3.10" - os: "windows-latest" - python-version: "3.8" + python-version: "3.9" steps: - uses: actions/checkout@v4 with: @@ -105,7 +104,7 @@ jobs: strategy: matrix: python_version: - - "3.8" + - "3.9" steps: - uses: actions/checkout@v4 with: @@ -243,14 +242,14 @@ jobs: CIBW_ENVIRONMENT_LINUX: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=/tmp/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" CIBW_ENVIRONMENT_WINDOWS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}\build_cache CIBW_ENVIRONMENT_MACOS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}/build_cache - CFLAGS="-g0 -DNDEBUG" + CFLAGS="-O3 -DNDEBUG" - name: Build sdist if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}} run: | diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 805a953a..508abc46 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,19 @@ Changelog .. that users understand how the changes affect the new version. +version 1.8.0-dev +----------------- ++ Python 3.8 is no longer supported. ++ Fix an issue where flushing using igzip_threaded caused a gzip end of stream + and started a new gzip stream. In essence creating a concatenated gzip + stream. Now it is in concordance with how single threaded gzip streams + are flushed using Z_SYNC_FLUSH. ++ Change build backend to setuptools-scm which is more commonly used and + supported. ++ Include test packages in the source distribution, so source distribution + installations can be verified. ++ Fix an issue where some tests failed because they ignored PYTHONPATH. + version 1.7.2 ----------------- + Use upstream ISA-L version 2.31.1 which includes patches to make diff --git a/MANIFEST.in b/MANIFEST.in index 8cc2b029..4a48e873 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,8 @@ graft src/isal/isa-l include src/isal/*.h -prune tests prune docs +prune .github +exclude .git* prune benchmark_scripts exclude requirements-docs.txt exclude codecov.yml diff --git a/pyproject.toml b/pyproject.toml index bf5fe99c..8405984e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,61 @@ [build-system] -requires = ["setuptools>=64", "versioningit>=1.1.0"] +requires = ["setuptools>=77", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" -[tool.versioningit.vcs] -method="git" -default-tag = "v0.0.0" +[project] +name = "isal" +dynamic = ["version"] +description = """ +Faster zlib and gzip compatible compression and decompression by providing \ +python bindings for the ISA-L ibrary.""" +license="PSF-2.0" +keywords=["isal", "isa-l", "compression", "deflate", "gzip", "igzip"] +authors = [{name = "Leiden University Medical Center"}, + {email = "r.h.p.vorderman@lumc.nl"}] +readme = "README.rst" +requires-python = ">=3.9" # Because of setuptools version +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: C", + "Development Status :: 5 - Production/Stable", + "Topic :: System :: Archiving :: Compression", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", +] +urls.homepage = "https://github.com/pycompression/python-isal" +urls.documentation = "python-isal.readthedocs.io" -[tool.versioningit.write] -file = "src/isal/_version.py" +[tool.setuptools_scm] +version_file = "src/isal/_version.py" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["isal"] + +[tool.setuptools.package-data] +isal = ['*.pyi', 'py.typed', 'isa-l/LICENSE', 'isa-l/README.md', 'isa-l/Release_notes.txt'] +[tool.setuptools.exclude-package-data] +isal = [ + "*.c", + "*.h", + "isa-l/*/*", + "isa-l/Mak*", + "isa-l/.*", + "isa-l/autogen.sh", + "isa-l/Doxyfile", + "isa-l/CONTRIBUTING.md", + "isa-l/SECURITY.md", + "isa-l/configure.ac", + "isa-l/isa-l.*", + "isa-l/libisal.pc.in", + "isa-l/make.inc", +] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 256134e1..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -license_files=LICENSE \ No newline at end of file diff --git a/setup.py b/setup.py index 55c89496..1ed64c2d 100644 --- a/setup.py +++ b/setup.py @@ -15,11 +15,9 @@ import tempfile from pathlib import Path -from setuptools import Extension, find_packages, setup +from setuptools import Extension, setup from setuptools.command.build_ext import build_ext -import versioningit - ISA_L_SOURCE = os.path.join("src", "isal", "isa-l") SYSTEM_IS_BSD = (sys.platform.startswith("freebsd") or @@ -138,46 +136,6 @@ def build_isa_l(): setup( - name="isal", - version=versioningit.get_version(), - description="Faster zlib and gzip compatible compression and " - "decompression by providing python bindings for the ISA-L " - "library.", - author="Leiden University Medical Center", - author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now - long_description=Path("README.rst").read_text(), - long_description_content_type="text/x-rst", cmdclass={"build_ext": BuildIsalExt}, - license="PSF-2.0", - keywords="isal isa-l compression deflate gzip igzip threads", - zip_safe=False, - packages=find_packages('src'), - package_dir={'': 'src'}, - package_data={'isal': ['*.pyi', 'py.typed', - # Include isa-l LICENSE and other relevant files - # with the binary distribution. - 'isa-l/LICENSE', 'isa-l/README.md', - 'isa-l/Release_notes.txt']}, - url="https://github.com/pycompression/python-isal", - classifiers=[ - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: C", - "Development Status :: 5 - Production/Stable", - "Topic :: System :: Archiving :: Compression", - "License :: OSI Approved :: Python Software Foundation License", - "Operating System :: POSIX :: Linux", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - ], - python_requires=">=3.8", # BadGzipFile imported ext_modules=EXTENSIONS ) diff --git a/src/isal/igzip_threaded.py b/src/isal/igzip_threaded.py index 7f1c94fc..355b7f72 100644 --- a/src/isal/igzip_threaded.py +++ b/src/isal/igzip_threaded.py @@ -316,7 +316,7 @@ def write(self, b) -> int: self.input_queues[worker_index].put((data, zdict)) return len(data) - def _end_gzip_stream(self): + def flush(self): self._check_closed() # Wait for all data to be compressed for in_q in self.input_queues: @@ -324,22 +324,18 @@ def _end_gzip_stream(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() - # Write an empty deflate block with a lost block marker. + self.raw.flush() + + def close(self) -> None: + if self._closed: + return + self.flush() self.raw.write(isal_zlib.compress(b"", wbits=-15)) trailer = struct.pack(" None: - if self._closed: - return - self._end_gzip_stream() self.stop() if self.exception: self.raw.close() diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index cc979317..5d71577c 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -845,9 +845,23 @@ def test_decompress_infile_outfile(self): self.assertTrue(os.path.exists(igzipname)) + # The following tests use assert_python_failure or assert_python_ok. + # + # If the env_vars argument to assert_python_failure or assert_python_ok + # is empty the test will run in isolated mode (-I) which means that the + # PYTHONPATH environment variable will be ignored and the test fails + # because the isal module can not be found, or the test is run using the + # system installed version of the module instead of the newly built + # module that should be tested. + # + # By adding a dummy entry to the env_vars argument the isolated mode is + # not used and the PYTHONPATH environment variable is not ignored and + # the test works as expected. + def test_decompress_infile_outfile_error(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '-d', - 'thisisatest.out') + 'thisisatest.out', + **{'_dummy': '1'}) self.assertEqual(b"filename doesn't end in .gz: 'thisisatest.out'. " b"Cannot determine output filename.", err.strip()) @@ -872,7 +886,8 @@ def test_compress_infile_outfile_default(self): with open(local_testigzip, 'wb') as fp: fp.write(self.data) - rc, out, err = assert_python_ok('-m', 'isal.igzip', local_testigzip) + rc, out, err = assert_python_ok('-m', 'isal.igzip', local_testigzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(igzipname)) self.assertEqual(out, b'') @@ -891,7 +906,8 @@ def test_compress_infile_outfile(self): rc, out, err = assert_python_ok('-m', 'isal.igzip', compress_level, - local_testigzip) + local_testigzip, + **{'_dummy': '1'}) self.assertTrue(os.path.exists(igzipname)) self.assertEqual(out, b'') @@ -901,7 +917,7 @@ def test_compress_infile_outfile(self): def test_compress_fast_best_are_exclusive(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '--fast', - '--best') + '--best', **{'_dummy': '1'}) self.assertIn( b"error: argument -3/--best: not allowed with argument -0/--fast", err) @@ -909,7 +925,7 @@ def test_compress_fast_best_are_exclusive(self): def test_decompress_cannot_have_flags_compression(self): rc, out, err = assert_python_failure('-m', 'isal.igzip', '--fast', - '-d') + '-d', **{'_dummy': '1'}) self.assertIn( b'error: argument -d/--decompress: not allowed with argument ' b'-0/--fast', diff --git a/tests/test_igzip_threaded.py b/tests/test_igzip_threaded.py index 41c61bfe..d2aee567 100644 --- a/tests/test_igzip_threaded.py +++ b/tests/test_igzip_threaded.py @@ -12,6 +12,7 @@ import subprocess import sys import tempfile +import zlib from pathlib import Path from isal import igzip_threaded @@ -243,15 +244,29 @@ def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): @pytest.mark.parametrize("threads", [1, 2]) def test_flush(tmp_path, threads): + empty_block_end = b"\x00\x00\xff\xff" + compressobj = zlib.compressobj(wbits=-15) + deflate_last_block = compressobj.compress(b"") + compressobj.flush() test_file = tmp_path / "output.gz" with igzip_threaded.open(test_file, "wb", threads=threads) as f: f.write(b"1") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"1" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + # Cut off gzip header and end data with an explicit last block to + # test if the data was compressed correctly. + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"1" f.write(b"2") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"12" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"12" f.write(b"3") f.flush() - assert gzip.decompress(test_file.read_bytes()) == b"123" + data = test_file.read_bytes() + assert data[-4:] == empty_block_end + deflate_block = data[10:] + deflate_last_block + assert zlib.decompress(deflate_block, wbits=-15) == b"123" assert gzip.decompress(test_file.read_bytes()) == b"123" pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy