diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 7b0c4dcfc4..a806c74ad9 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.10, 3.11, 3.12 etc. + placeholder: 3.11, 3.12, 3.13 etc. validations: required: true - type: input diff --git a/.github/workflows/latest-deps.yml b/.github/workflows/latest-deps.yml new file mode 100644 index 0000000000..1ae16a53fc --- /dev/null +++ b/.github/workflows/latest-deps.yml @@ -0,0 +1,30 @@ +# This workflow simulates a fresh environment where the only install +# command is a user pip installing zarr +name: Latest dependencies + +on: + push: + branches: [ support/v2 ] + pull_request: + branches: [ support/v2 ] + +jobs: + latest-deps: + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v3.2.0 + with: + auto-update-conda: true + python-version: "3.12" + - name: Tests + shell: bash -el {0} + env: + ZARR_V3_EXPERIMENTAL_API: 1 + ZARR_V3_SHARDING: 1 + run: | + python -m pip install pytest pytest-cov pytest-timeout msgpack-python + python -m pip install . + pytest -svx --timeout=300 diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index b5b2f48d62..f94d5d373f 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -4,9 +4,9 @@ name: Minimal installation on: push: - branches: [ main ] + branches: [ support/v2 ] pull_request: - branches: [ main ] + branches: [ support/v2 ] jobs: minimum_build: @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.4 + uses: conda-incubator/setup-miniconda@v3.2.0 with: channels: conda-forge environment-file: environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b1be7e425d..60367a7b20 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,9 +5,9 @@ name: Linux Testing on: push: - branches: [ main ] + branches: [ support/v2] pull_request: - branches: [ main ] + branches: [ support/v2] jobs: build: @@ -15,11 +15,19 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.11', '3.12', '3.13'] numpy_version: ['>=2.1', '==1.24.*'] + condadeps: [''] exclude: - python-version: '3.12' numpy_version: '==1.24.*' + - python-version: '3.13' + numpy_version: '==1.24.*' + include: + - python-version: '3.13' + numpy_version: '>=2.1' + # to test alternate hexdigests with zlib-ng + condadeps: 'zlib-ng' services: redis: image: redis @@ -40,7 +48,7 @@ jobs: with: fetch-depth: 0 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.4 + uses: conda-incubator/setup-miniconda@v3.2.0 with: channels: conda-forge python-version: ${{ matrix.python-version }} @@ -50,7 +58,7 @@ jobs: - name: Create Conda environment with the rights deps shell: "bash -l {0}" run: | - conda create -n zarr-env python==${{matrix.python-version}} bsddb3 pip nodejs + conda create -n zarr-env python==${{matrix.python-version}} pip nodejs ${{matrix.condadeps}} conda activate zarr-env npm install -g azurite - name: Install dependencies @@ -75,7 +83,7 @@ jobs: mkdir ~/blob_emulator azurite -l ~/blob_emulator --debug debug.log 2>&1 > stdouterr.log & pytest --cov=zarr --cov-config=pyproject.toml --doctest-plus --cov-report xml --cov=./ --timeout=300 - - uses: codecov/codecov-action@v4 + - uses: codecov/codecov-action@v5 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 26b669abea..7b93f572ad 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.2.0 + - uses: actions/setup-python@v5.6.0 name: Install Python with: python-version: '3.11' @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.10.3 + - uses: pypa/gh-action-pypi-publish@v1.12.4 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index 1e22fec6d1..7fb1be5383 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -5,9 +5,9 @@ name: Python package on: push: - branches: [ main ] + branches: [ support/v2 ] pull_request: - branches: [ main ] + branches: [ support/v2 ] jobs: windows: @@ -16,12 +16,12 @@ jobs: strategy: fail-fast: True matrix: - python-version: ['3.10', '3.11'] + python-version: ['3.11', '3.12'] steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3.0.4 + - uses: conda-incubator/setup-miniconda@v3.2.0 with: auto-update-conda: true python-version: ${{ matrix.python-version }} diff --git a/.readthedocs.yaml b/.readthedocs.yaml index d7190b4771..d407808e78 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,6 +4,9 @@ build: os: ubuntu-20.04 tools: python: "3.11" + jobs: + post_checkout: + - git fetch --unshallow || true sphinx: configuration: docs/conf.py diff --git a/docs/conf.py b/docs/conf.py index 136fcf32d6..31dc7da204 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,6 +44,7 @@ "sphinx_issues", "sphinx_copybutton", "sphinx_design", + "pytest_doctestplus.sphinx.doctestplus", ] numpydoc_show_class_members = False @@ -72,8 +73,8 @@ copyright = "2024, Zarr Developers" author = "Zarr Developers" -version = get_version("zarr") release = get_version("zarr") +version = get_version("zarr") # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/release.rst b/docs/release.rst index a62d6a653c..56bbd0d2f1 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -14,23 +14,66 @@ Release notes # re-indented so that it does not show up in the notes. .. note:: - Zarr-Python 2.18.* is expected be the final release in the 2.* series. Work on Zarr-Python 3.0 is underway. - See `GH1777 `_ for more details on the upcoming - 3.0 release. + Zarr-Python 2 is in support mode now, and no new features will be added. + +.. _release_2.18.7: + +2.18.7 +------ + +Fixes +~~~~~ +* Pinned ``numcodecs`` to ``<0.16``. In ``numcodecs`` 0.16 deprecated code was removed + that makes it incompatible with older versions of zarr-python 2.18. + By :user:`David Stansby ` (:issue:`2973`) + +.. _release_2.18.6: + +2.18.6 +------ +Note: the numcodecs dependency pin was incorrectly applied in this release, meaning it maintains +the same issue with numcodecs compatibility as previous releases. Please upgrade +to 2.18.7 to fix this. + +.. _release_2.18.5: + +2.18.5 +------ + +Fixes +~~~~~ +* Fixed ``SQLiteStore`` with the latest version of ``sqlite3``. + By :user:`David Stansby ` + +Deprecations +~~~~~~~~~~~~ + +* Deprecated support for ``partial_decompress`` when creating an array. + This functionality is no longer supported in ``numcodecs``, and will be removed + in ``zarr-python`` 2.19.0. + By :user:`David Stansby ` .. _release_2.18.4: -2.18.4 (unreleased) -------------------- +2.18.4 +------ Enhancements ~~~~~~~~~~~~ +* Added testing on Python 3.13. Maintenance ~~~~~~~~~~~ - -Deprecations -~~~~~~~~~~~~ +* Dropped support for Python 3.10. + By :user:`David Stansby ` (:issue:`2344`). +* Removed testing for compatibility with the ``bsddb3`` package. + ``bsddb3`` was last released four years ago, and should still work with + ``zarr-python`` if you can install it, but to reduce our maintenance + burden we will no longer run our compatibility tests for it. + By :user:`David Stansby ` (:issue:`2344`). +* Excluded versions 0.14.0 and 0.14.1 of numcodecs, due to a bug in the implementation of + the Delta filter (see https://github.com/zarr-developers/numcodecs/issues/653 for more information). + By :user:`David Stansby ` (:issue:`2544`). .. _release_2.18.3: @@ -41,7 +84,7 @@ Enhancements ~~~~~~~~~~~~ * Added support for creating a copy of data when converting a `zarr.Array` to a numpy array. - By :user:`David Stansby ` (:issue:`2106`) and + By :user:`David Stansby ` (:issue:`2106`) and :user:`Joe Hamman ` (:issue:`2123`). Maintenance diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 214dd4f63f..b94cf3fa1c 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -809,7 +809,9 @@ Another storage alternative is the :class:`zarr.storage.DBMStore` class, added in Zarr version 2.2. This class allows any DBM-style database to be used for storing an array or group. Here is an example using a Berkeley DB B-tree database for storage (requires `bsddb3 -`_ to be installed):: +`_ to be installed): + +.. doctest-requires:: bsddb3 >>> import bsddb3 >>> store = zarr.DBMStore('data/example.bdb', open=bsddb3.btopen) @@ -1009,12 +1011,12 @@ class from ``fsspec``. The following example demonstrates how to access a ZIP-archived Zarr group on s3 using `s3fs `_ and ``ZipFileSystem``: >>> s3_path = "s3://path/to/my.zarr.zip" - >>> + >>> >>> s3 = s3fs.S3FileSystem() >>> f = s3.open(s3_path) >>> fs = ZipFileSystem(f, mode="r") >>> store = FSMap("", fs, check=False) - >>> + >>> >>> # caching may improve performance when repeatedly reading the same data >>> cache = zarr.storage.LRUStoreCache(store, max_size=2**28) >>> z = zarr.group(store=cache) diff --git a/pyproject.toml b/pyproject.toml index ec06b63a96..02789ba0b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=64.0.0", "setuptools-scm>1.5.4"] +requires = ["setuptools>=64", "setuptools-scm>8"] build-backend = "setuptools.build_meta" @@ -7,19 +7,15 @@ build-backend = "setuptools.build_meta" name = "zarr" description = "An implementation of chunked, compressed, N-dimensional arrays for Python" readme = { file = "README.md", content-type = "text/markdown" } -maintainers = [ - { name = "Alistair Miles", email = "alimanfoo@googlemail.com" } -] -requires-python = ">=3.10" +maintainers = [{ name = "Alistair Miles", email = "alimanfoo@googlemail.com" }] +requires-python = ">=3.11" dependencies = [ 'asciitree', 'numpy>=1.24', 'fasteners; sys_platform != "emscripten"', - 'numcodecs>=0.10.0', -] -dynamic = [ - "version", + 'numcodecs>=0.10.0,!=0.14.0,!=0.14.1,<0.16', ] +dynamic = ["version"] classifiers = [ 'Development Status :: 6 - Mature', 'Intended Audience :: Developers', @@ -30,18 +26,14 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', ] license = { text = "MIT" } [project.optional-dependencies] -jupyter = [ - 'notebook', - 'ipytree>=0.2.2', - 'ipywidgets>=8.0.0', -] +jupyter = ['notebook', 'ipytree>=0.2.2', 'ipywidgets>=8.0.0'] docs = [ 'sphinx', 'sphinx-automodapi', @@ -50,7 +42,8 @@ docs = [ 'sphinx-copybutton', 'pydata-sphinx-theme', 'numpydoc', - 'numcodecs[msgpack]', + 'numcodecs[msgpack]!=0.14.0,!=0.14.1,<0.16', + 'pytest-doctestplus', ] [project.urls] @@ -64,23 +57,19 @@ Homepage = "https://github.com/zarr-developers/zarr-python" exclude_lines = [ "pragma: no cover", "pragma: ${PY_MAJOR_VERSION} no cover", - '.*\.\.\.' # Ignore "..." lines + '.*\.\.\.', # Ignore "..." lines ] [tool.coverage.run] -omit = [ - "zarr/meta_v1.py", - "bench/compress_normal.py", -] +omit = ["zarr/meta_v1.py", "bench/compress_normal.py"] [tool.setuptools] packages = ["zarr", "zarr._storage", "zarr.tests"] license-files = ["LICENSE.txt"] [tool.setuptools_scm] -version_scheme = "guess-next-dev" local_scheme = "dirty-tag" -write_to = "zarr/version.py" +version_file = "zarr/version.py" [tool.ruff] line-length = 100 @@ -100,14 +89,12 @@ exclude = [ "build", "dist", "venv", - "docs" + "docs", ] [tool.ruff.lint] -extend-select = [ - "B" -] -ignore = ["B905"] # zip-without-explicit-strict +extend-select = ["B"] +ignore = ["B905"] # zip-without-explicit-strict [tool.black] line-length = 100 @@ -136,19 +123,18 @@ doctest_optionflags = [ "ELLIPSIS", "IGNORE_EXCEPTION_DETAIL", ] -addopts = [ - "--durations=10", -] +addopts = ["--durations=10"] filterwarnings = [ "error:::zarr.*", "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", "ignore:The .* is deprecated and will be removed in a Zarr-Python version 3*:FutureWarning", "ignore:The experimental Zarr V3 implementation in this version .*:FutureWarning", + "ignore:unclosed database in =2", - "zarr/creation.py = numpy>=2" + "zarr/creation.py = numpy>=2", ] diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index caa078cc82..e7805123ee 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -1,8 +1,8 @@ # library requirements asciitree==0.3.3 fasteners==0.19 -numcodecs==0.13.1 +numcodecs==0.15.1 msgpack-python==0.5.6 -setuptools-scm==8.1.0 +setuptools-scm==8.3.1 # test requirements -pytest==8.3.3 +pytest==8.4.1 diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index 4a619aa3ef..2c8f1ab65c 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==2.1.2 +numpy==2.3.1 diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index df1d4fd793..fc4c5666a3 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -1,23 +1,20 @@ # optional library requirements # bsddb3==6.2.6; sys_platform != 'win32' -lmdb==1.5.1; sys_platform != 'win32' +lmdb==1.7.2; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.5 -# optional library requirements for services -# don't let pyup change pinning for azure-storage-blob, need to pin to older -# version to get compatibility with azure storage emulator on appveyor (FIXME) -azure-storage-blob==12.21.0 # pyup: ignore -redis==5.1.1 +ipywidgets==8.1.7 +azure-storage-blob==12.24.1 +redis==6.2.0 types-redis types-setuptools -pymongo==4.10.1 +pymongo==4.13.2 # optional test requirements coverage -pytest-cov==5.0.0 -pytest-doctestplus==1.2.1 -pytest-timeout==2.3.1 -h5py==3.12.1 +pytest-cov==6.2.1 +pytest-doctestplus==1.4.0 +pytest-timeout==2.4.0 +h5py==3.14.0 fsspec==2023.12.2 s3fs==2023.12.2 moto[server]>=5.0.1 diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index 1e49754f38..4fbb4078eb 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -231,7 +231,7 @@ def getsize(self, path=None): elif not fs_path.endswith("/"): fs_path += "/" for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter="/"): - blob_client = self.client.get_blob_client(blob) + blob_client = self.client.get_blob_client(blob.name) if blob_client.exists(): size += blob_client.get_blob_properties().size return size diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 4987f820cf..334788585f 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -490,7 +490,7 @@ def rmdir(self, path=None): if path: for base in [meta_root, data_root]: with self.lock: - self.cursor.execute('DELETE FROM zarr WHERE k LIKE (? || "/%")', (base + path,)) + self.cursor.execute("DELETE FROM zarr WHERE k LIKE (? || '/%')", (base + path,)) # remove any associated metadata files sfx = _get_metadata_suffix(self) meta_dir = (meta_root + path).rstrip("/") diff --git a/zarr/convenience.py b/zarr/convenience.py index bd284e0844..a3cd702c9d 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -31,6 +31,8 @@ _builtin_open = open # builtin open is later shadowed by a local open function +__doctest_requires__ = {("*"): ["numpy>=2.2"]} + def _check_and_update_path(store: BaseStore, path): if getattr(store, "_store_version", 2) > 2 and not path: @@ -174,13 +176,13 @@ def save_array(store: StoreLike, arr, *, zarr_version=None, path=None, **kwargs) >>> arr = np.arange(10000) >>> zarr.save_array('data/example.zarr', arr) >>> zarr.load('data/example.zarr') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) + array([ 0, 1, 2, ..., 9997, 9998, 9999], shape=(10000,)) Save an array to a single file (uses a :class:`ZipStore`):: >>> zarr.save_array('data/example.zip', arr) >>> zarr.load('data/example.zip') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) + array([ 0, 1, 2, ..., 9997, 9998, 9999], shape=(10000,)) """ may_need_closing = _might_close(store) @@ -234,9 +236,9 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): >>> loader >>> loader['arr_0'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) + array([ 0, 1, 2, ..., 9997, 9998, 9999], shape=(10000,)) >>> loader['arr_1'] - array([10000, 9999, 9998, ..., 3, 2, 1]) + array([10000, 9999, 9998, ..., 3, 2, 1], shape=(10000,)) Save several arrays using named keyword arguments:: @@ -245,9 +247,9 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): >>> loader >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) + array([ 0, 1, 2, ..., 9997, 9998, 9999], shape=(10000,)) >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) + array([10000, 9999, 9998, ..., 3, 2, 1], shape=(10000,)) Store several arrays in a single zip file (uses a :class:`ZipStore`):: @@ -256,9 +258,9 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): >>> loader >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) + array([ 0, 1, 2, ..., 9997, 9998, 9999], shape=(10000,)) >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) + array([10000, 9999, 9998, ..., 3, 2, 1], shape=(10000,)) Notes ----- @@ -316,13 +318,13 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): >>> arr = np.arange(10000) >>> zarr.save('data/example.zarr', arr) >>> zarr.load('data/example.zarr') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) + array([ 0, 1, 2, ..., 9997, 9998, 9999], shape=(10000,)) Save an array to a Zip file (uses a :class:`ZipStore`):: >>> zarr.save('data/example.zip', arr) >>> zarr.load('data/example.zip') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) + array([ 0, 1, 2, ..., 9997, 9998, 9999], shape=(10000,)) Save several arrays to a directory on the file system (uses a :class:`DirectoryStore` and stores arrays in a group):: @@ -336,9 +338,9 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): >>> loader >>> loader['arr_0'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) + array([ 0, 1, 2, ..., 9997, 9998, 9999], shape=(10000,)) >>> loader['arr_1'] - array([10000, 9999, 9998, ..., 3, 2, 1]) + array([10000, 9999, 9998, ..., 3, 2, 1], shape=(10000,)) Save several arrays using named keyword arguments:: @@ -347,9 +349,9 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): >>> loader >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) + array([ 0, 1, 2, ..., 9997, 9998, 9999], shape=(10000,)) >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) + array([10000, 9999, 9998, ..., 3, 2, 1], shape=(10000,)) Store several arrays in a single zip file (uses a :class:`ZipStore`):: @@ -358,9 +360,9 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): >>> loader >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) + array([ 0, 1, 2, ..., 9997, 9998, 9999], shape=(10000,)) >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) + array([10000, 9999, 9998, ..., 3, 2, 1], shape=(10000,)) See Also -------- diff --git a/zarr/core.py b/zarr/core.py index d13da27bc6..0bbea83816 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -6,6 +6,7 @@ import re from functools import reduce from typing import Any +import warnings import numpy as np from numcodecs.compat import ensure_bytes @@ -61,6 +62,7 @@ ) __all__ = ["Array"] +__doctest_requires__ = {("*"): ["numpy>=2.2"]} # noinspection PyUnresolvedReferences @@ -89,13 +91,6 @@ class Array: If True (default), user attributes will be cached for attribute read operations. If False, user attributes are reloaded from the store prior to all attribute read operations. - partial_decompress : bool, optional - If True and while the chunk_store is a FSStore and the compression used - is Blosc, when getting data from the array chunks will be partially - read and decompressed when possible. - - .. versionadded:: 2.7 - write_empty_chunks : bool, optional If True, all chunks will be stored regardless of their contents. If False (default), each chunk is compared to the array's fill value prior @@ -123,7 +118,7 @@ def __init__( synchronizer=None, cache_metadata=True, cache_attrs=True, - partial_decompress=False, + partial_decompress=None, write_empty_chunks=True, zarr_version=None, meta_array=None, @@ -153,6 +148,13 @@ def __init__( self._synchronizer = synchronizer self._cache_metadata = cache_metadata self._is_view = False + if partial_decompress is not None: + warnings.warn( + "Support for partial decompression is no longer supported in numcodecs. " + "Support for partial decompression will be removed in a future version of zarr-python v2.", + DeprecationWarning, + stacklevel=1, + ) self._partial_decompress = partial_decompress self._write_empty_chunks = write_empty_chunks if meta_array is not None: @@ -2793,12 +2795,12 @@ def view( >>> a = zarr.array(data, chunks=1000, filters=filters) >>> a[:] array(['female', 'male', 'female', ..., 'male', 'male', 'female'], - dtype='>> v = a.view(dtype='u1', filters=[]) >>> v.is_view True >>> v[:] - array([1, 2, 1, ..., 2, 2, 1], dtype=uint8) + array([1, 2, 1, ..., 2, 2, 1], shape=(10000,), dtype=uint8) Views can be used to modify data: @@ -2806,20 +2808,20 @@ def view( >>> x.sort() >>> v[:] = x >>> v[:] - array([1, 1, 1, ..., 2, 2, 2], dtype=uint8) + array([1, 1, 1, ..., 2, 2, 2], shape=(10000,), dtype=uint8) >>> a[:] array(['female', 'female', 'female', ..., 'male', 'male', 'male'], - dtype='>> data = np.random.randint(0, 2, size=10000, dtype='u1') >>> a = zarr.array(data, chunks=1000) >>> a[:] - array([0, 0, 1, ..., 1, 0, 0], dtype=uint8) + array([0, 0, 1, ..., 1, 0, 0], shape=(10000,), dtype=uint8) >>> v = a.view(dtype=bool) >>> v[:] - array([False, False, True, ..., True, False, False]) + array([False, False, True, ..., True, False, False], shape=(10000,)) >>> np.all(a[:].view(dtype=bool) == v[:]) np.True_ @@ -2841,10 +2843,10 @@ def view( >>> a = zarr.full(10000, chunks=1000, fill_value=-1, dtype='i1') >>> a[:] - array([-1, -1, -1, ..., -1, -1, -1], dtype=int8) + array([-1, -1, -1, ..., -1, -1, -1], shape=(10000,), dtype=int8) >>> v = a.view(fill_value=42) >>> v[:] - array([42, 42, 42, ..., 42, 42, 42], dtype=int8) + array([42, 42, 42, ..., 42, 42, 42], shape=(10000,), dtype=int8) Note that resizing or appending to views is not permitted: diff --git a/zarr/creation.py b/zarr/creation.py index f7f3d5a094..e54fb408f8 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -466,7 +466,7 @@ def open_array( object_codec=None, chunk_store=None, storage_options=None, - partial_decompress=False, + partial_decompress=None, write_empty_chunks=True, *, zarr_version=None, @@ -522,10 +522,6 @@ def open_array( storage_options : dict If using an fsspec URL to create the store, these will be passed to the backend implementation. Ignored otherwise. - partial_decompress : bool, optional - If True and while the chunk_store is a FSStore and the compression used - is Blosc, when getting data from the array chunks will be partially - read and decompressed when possible. write_empty_chunks : bool, optional If True (default), all chunks will be stored regardless of their contents. If False, each chunk is compared to the array's fill value diff --git a/zarr/storage.py b/zarr/storage.py index f412870f75..f9f6dbe0a6 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1686,7 +1686,6 @@ class NestedDirectoryStore(DirectoryStore): def __init__( self, path, normalize_keys=False, dimension_separator: Optional[DIMENSION_SEPARATOR] = "/" ): - warnings.warn( V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), FutureWarning, @@ -2070,11 +2069,11 @@ class DBMStore(Store): `_ package is installed, a Berkeley DB database can be used:: - >>> import bsddb3 - >>> store = zarr.DBMStore('data/array.bdb', open=bsddb3.btopen) - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - >>> store.close() + >>> import bsddb3 # doctest: +SKIP + >>> store = zarr.DBMStore('data/array.bdb', open=bsddb3.btopen) # doctest: +SKIP + >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) # doctest: +SKIP + >>> z[...] = 42 # doctest: +SKIP + >>> store.close() # doctest: +SKIP Notes ----- @@ -2780,9 +2779,9 @@ def listdir(self, path=None): sep = "_" if path == "" else "/" keys = self.cursor.execute( f""" - SELECT DISTINCT SUBSTR(m, 0, INSTR(m, "/")) AS l FROM ( - SELECT LTRIM(SUBSTR(k, LENGTH(?) + 1), "/") || "/" AS m - FROM zarr WHERE k LIKE (? || "{sep}%") + SELECT DISTINCT SUBSTR(m, 0, INSTR(m, '/')) AS l FROM ( + SELECT LTRIM(SUBSTR(k, LENGTH(?) + 1), '/') || '/' AS m + FROM zarr WHERE k LIKE (? || '{sep}%') ) ORDER BY l ASC """, (path, path), @@ -2795,8 +2794,8 @@ def getsize(self, path=None): size = self.cursor.execute( """ SELECT COALESCE(SUM(LENGTH(v)), 0) FROM zarr - WHERE k LIKE (? || "%") AND - 0 == INSTR(LTRIM(SUBSTR(k, LENGTH(?) + 1), "/"), "/") + WHERE k LIKE (? || '%') AND + 0 == INSTR(LTRIM(SUBSTR(k, LENGTH(?) + 1), '/'), '/') """, (path, path), ) @@ -2807,7 +2806,7 @@ def rmdir(self, path=None): path = normalize_storage_path(path) if path: with self.lock: - self.cursor.execute('DELETE FROM zarr WHERE k LIKE (? || "/%")', (path,)) + self.cursor.execute("DELETE FROM zarr WHERE k LIKE (? || '/%')", (path,)) else: self.clear() diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 4729dc01b6..a4e5a5e912 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -1,8 +1,8 @@ import atexit -import os import sys import pickle import shutil + from typing import Any, Literal, Optional, Tuple, Union, Sequence import unittest from itertools import zip_longest @@ -75,7 +75,6 @@ from zarr.util import buffer_size from zarr.tests.util import ( abs_container, - have_bsddb3, have_fsspec, have_lmdb, have_sqlite3, @@ -86,6 +85,11 @@ # noinspection PyMethodMayBeStatic +pytestmark = [ + pytest.mark.filterwarnings("ignore:Call to deprecated function .* \_cbuffer\_sizes.*"), + pytest.mark.filterwarnings("ignore:Call to deprecated function .* \_cbuffer\_metainfo.*"), +] + class TestArray: version = 2 @@ -96,7 +100,7 @@ class TestArray: dimension_separator: Optional[DIMENSION_SEPARATOR] = None cache_metadata = True cache_attrs = True - partial_decompress: bool = False + partial_decompress: bool | None = None write_empty_chunks = True read_only = False storage_transformers: Tuple[Any, ...] = () @@ -626,13 +630,24 @@ def test_setitem_data_not_shared(self): def expected(self): # tests for array without path will not be run for v3 stores assert self.version == 2 - return [ - "063b02ff8d9d3bab6da932ad5828b506ef0a6578", - "f97b84dc9ffac807415f750100108764e837bb82", - "c7190ad2bea1e9d2e73eaa2d3ca9187be1ead261", - "14470724dca6c1837edddedc490571b6a7f270bc", - "2a1046dd99b914459b3e86be9dde05027a07d209", - ] + return ( + # zlib + [ + "063b02ff8d9d3bab6da932ad5828b506ef0a6578", + "f97b84dc9ffac807415f750100108764e837bb82", + "c7190ad2bea1e9d2e73eaa2d3ca9187be1ead261", + "14470724dca6c1837edddedc490571b6a7f270bc", + "2a1046dd99b914459b3e86be9dde05027a07d209", + ], + # zlib-ng + [ + "063b02ff8d9d3bab6da932ad5828b506ef0a6578", + "f97b84dc9ffac807415f750100108764e837bb82", + "c7190ad2bea1e9d2e73eaa2d3ca9187be1ead261", + "f3f04f0e30844739d34ef8a9eee6c949a47840b8", + "2a1046dd99b914459b3e86be9dde05027a07d209", + ], + ) def test_hexdigest(self): found = [] @@ -671,7 +686,7 @@ def test_hexdigest(self): found.append(z.hexdigest()) z.store.close() - assert self.expected() == found + assert found in self.expected() def test_resize_1d(self): z = self.create_array(shape=105, chunks=10, dtype="i4", fill_value=0) @@ -1620,13 +1635,15 @@ def test_nchunks_initialized(self): pass def expected(self): - return [ - "f710da18d45d38d4aaf2afd7fb822fdd73d02957", - "1437428e69754b1e1a38bd7fc9e43669577620db", - "6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe", - "4c0a76fb1222498e09dcd92f7f9221d6cea8b40e", - "05b0663ffe1785f38d3a459dec17e57a18f254af", - ] + return ( + [ + "f710da18d45d38d4aaf2afd7fb822fdd73d02957", + "1437428e69754b1e1a38bd7fc9e43669577620db", + "6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe", + "4c0a76fb1222498e09dcd92f7f9221d6cea8b40e", + "05b0663ffe1785f38d3a459dec17e57a18f254af", + ], + ) def test_nbytes_stored(self): # MemoryStore as store @@ -1653,13 +1670,15 @@ def create_chunk_store(self): return KVStore(dict()) def expected(self): - return [ - "f710da18d45d38d4aaf2afd7fb822fdd73d02957", - "1437428e69754b1e1a38bd7fc9e43669577620db", - "6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe", - "4c0a76fb1222498e09dcd92f7f9221d6cea8b40e", - "05b0663ffe1785f38d3a459dec17e57a18f254af", - ] + return ( + [ + "f710da18d45d38d4aaf2afd7fb822fdd73d02957", + "1437428e69754b1e1a38bd7fc9e43669577620db", + "6c530b6b9d73e108cc5ee7b6be3d552cc994bdbe", + "4c0a76fb1222498e09dcd92f7f9221d6cea8b40e", + "05b0663ffe1785f38d3a459dec17e57a18f254af", + ], + ) def test_nbytes_stored(self): z = self.create_array(shape=1000, chunks=100) @@ -1729,13 +1748,24 @@ def create_store(self): return store def expected(self): - return [ - "d174aa384e660eb51c6061fc8d20850c1159141f", - "125f00eea40032f16016b292f6767aa3928c00a7", - "1b52ead0ed889a781ebd4db077a29e35d513c1f3", - "719a88b34e362ff65df30e8f8810c1146ab72bc1", - "6e0abf30daf45de51593c227fb907759ca725551", - ] + return ( + # zlib + [ + "d174aa384e660eb51c6061fc8d20850c1159141f", + "125f00eea40032f16016b292f6767aa3928c00a7", + "1b52ead0ed889a781ebd4db077a29e35d513c1f3", + "719a88b34e362ff65df30e8f8810c1146ab72bc1", + "6e0abf30daf45de51593c227fb907759ca725551", + ], + # zlib-ng + [ + "d174aa384e660eb51c6061fc8d20850c1159141f", + "125f00eea40032f16016b292f6767aa3928c00a7", + "1b52ead0ed889a781ebd4db077a29e35d513c1f3", + "42d9c96e60ed22346c4671bc5bec32a2078ce25c", + "6e0abf30daf45de51593c227fb907759ca725551", + ], + ) class TestArrayWithN5Store(TestArrayWithDirectoryStore): @@ -2017,13 +2047,24 @@ def test_compressors(self): assert np.all(a2[:] == 1) def expected(self): - return [ - "8811a77d54caaa1901d5cc4452d946ae433c8d90", - "d880b007d9779db5f2cdbe13274eb1cbac4a425a", - "d80eb66d5521744f051e816ab368d8ccfc2e3edf", - "568f9f837e4b682a3819cb122988e2eebeb6572b", - "4fdf4475d786d6694110db5619acd30c80dfc372", - ] + return ( + # zlib + [ + "8811a77d54caaa1901d5cc4452d946ae433c8d90", + "d880b007d9779db5f2cdbe13274eb1cbac4a425a", + "d80eb66d5521744f051e816ab368d8ccfc2e3edf", + "568f9f837e4b682a3819cb122988e2eebeb6572b", + "4fdf4475d786d6694110db5619acd30c80dfc372", + ], + # zlib-ng + [ + "8811a77d54caaa1901d5cc4452d946ae433c8d90", + "d880b007d9779db5f2cdbe13274eb1cbac4a425a", + "d80eb66d5521744f051e816ab368d8ccfc2e3edf", + "ea7d9e80211679291141840b111775b088e51480", + "4fdf4475d786d6694110db5619acd30c80dfc372", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @@ -2046,20 +2087,6 @@ def test_nbytes_stored(self): pass # not implemented -@pytest.mark.skipif(have_bsddb3 is False, reason="needs bsddb3") -class TestArrayWithDBMStoreBerkeleyDB(TestArray): - def create_store(self): - import bsddb3 - - path = mktemp(suffix=".dbm") - atexit.register(os.remove, path) - store = DBMStore(path, flag="n", open=bsddb3.btopen) - return store - - def test_nbytes_stored(self): - pass # not implemented - - @pytest.mark.skipif(have_lmdb is False, reason="needs lmdb") class TestArrayWithLMDBStore(TestArray): def create_store(self): @@ -2103,39 +2130,45 @@ class TestArrayWithNoCompressor(TestArray): compressor = None def expected(self): - return [ - "d3da3d485de4a5fcc6d91f9dfc6a7cba9720c561", - "443b8dee512e42946cb63ff01d28e9bee8105a5f", - "b75eb90f68aa8ee1e29f2c542e851d3945066c54", - "42b6ae0d50ec361628736ab7e68fe5fefca22136", - "a0535f31c130f5e5ac66ba0713d1c1ceaebd089b", - ] + return ( + [ + "d3da3d485de4a5fcc6d91f9dfc6a7cba9720c561", + "443b8dee512e42946cb63ff01d28e9bee8105a5f", + "b75eb90f68aa8ee1e29f2c542e851d3945066c54", + "42b6ae0d50ec361628736ab7e68fe5fefca22136", + "a0535f31c130f5e5ac66ba0713d1c1ceaebd089b", + ], + ) class TestArrayWithBZ2Compressor(TestArray): compressor = BZ2(level=1) def expected(self): - return [ - "33141032439fb1df5e24ad9891a7d845b6c668c8", - "44d719da065c88a412d609a5500ff41e07b331d6", - "37c7c46e5730bba37da5e518c9d75f0d774c5098", - "1e1bcaac63e4ef3c4a68f11672537131c627f168", - "86d7b9bf22dccbeaa22f340f38be506b55e76ff2", - ] + return ( + [ + "33141032439fb1df5e24ad9891a7d845b6c668c8", + "44d719da065c88a412d609a5500ff41e07b331d6", + "37c7c46e5730bba37da5e518c9d75f0d774c5098", + "1e1bcaac63e4ef3c4a68f11672537131c627f168", + "86d7b9bf22dccbeaa22f340f38be506b55e76ff2", + ], + ) class TestArrayWithBloscCompressor(TestArray): compressor = Blosc(cname="zstd", clevel=1, shuffle=1) def expected(self): - return [ - "7ff2ae8511eac915fad311647c168ccfe943e788", - "962705c861863495e9ccb7be7735907aa15e85b5", - "74ed339cfe84d544ac023d085ea0cd6a63f56c4b", - "90e30bdab745a9641cd0eb605356f531bc8ec1c3", - "95d40c391f167db8b1290e3c39d9bf741edacdf6", - ] + return ( + [ + "7ff2ae8511eac915fad311647c168ccfe943e788", + "962705c861863495e9ccb7be7735907aa15e85b5", + "74ed339cfe84d544ac023d085ea0cd6a63f56c4b", + "90e30bdab745a9641cd0eb605356f531bc8ec1c3", + "95d40c391f167db8b1290e3c39d9bf741edacdf6", + ], + ) try: @@ -2149,13 +2182,15 @@ class TestArrayWithLZMACompressor(TestArray): compressor = LZMA(preset=1) def expected(self): - return [ - "93ecaa530a1162a9d48a3c1dcee4586ccfc59bae", - "04a9755a0cd638683531b7816c7fa4fbb6f577f2", - "9de97b5c49b38e68583ed701d7e8f4c94b6a8406", - "cde499f3dc945b4e97197ff8e3cf8188a1262c35", - "e2cf3afbf66ad0e28a2b6b68b1b07817c69aaee2", - ] + return ( + [ + "93ecaa530a1162a9d48a3c1dcee4586ccfc59bae", + "04a9755a0cd638683531b7816c7fa4fbb6f577f2", + "9de97b5c49b38e68583ed701d7e8f4c94b6a8406", + "cde499f3dc945b4e97197ff8e3cf8188a1262c35", + "e2cf3afbf66ad0e28a2b6b68b1b07817c69aaee2", + ], + ) class TestArrayWithFilters(TestArray): @@ -2168,13 +2203,24 @@ def create_filters(self, dtype: Optional[str]) -> Tuple[Any, ...]: ) def expected(self): - return [ - "b80367c5599d47110d42bd8886240c2f46620dba", - "95a7b2471225e73199c9716d21e8d3dd6e5f6f2a", - "7300f1eb130cff5891630038fd99c28ef23d3a01", - "c649ad229bc5720258b934ea958570c2f354c2eb", - "62fc9236d78af18a5ec26c12eea1d33bce52501e", - ] + return ( + # zlib + [ + "b80367c5599d47110d42bd8886240c2f46620dba", + "95a7b2471225e73199c9716d21e8d3dd6e5f6f2a", + "7300f1eb130cff5891630038fd99c28ef23d3a01", + "c649ad229bc5720258b934ea958570c2f354c2eb", + "62fc9236d78af18a5ec26c12eea1d33bce52501e", + ], + # zlib-ng + [ + "b80367c5599d47110d42bd8886240c2f46620dba", + "95a7b2471225e73199c9716d21e8d3dd6e5f6f2a", + "7300f1eb130cff5891630038fd99c28ef23d3a01", + "1e053b6ad7dc58de7b1f5dad7fb45851f6b7b3ee", + "62fc9236d78af18a5ec26c12eea1d33bce52501e", + ], + ) def test_astype_no_filters(self): shape = (100,) @@ -2299,7 +2345,8 @@ def test_nbytes_stored(self): z = self.create_array(shape=1000, chunks=100) assert 245 == z.nbytes_stored z[:] = 42 - assert 515 == z.nbytes_stored + # 515 is zlib, 485 is zlib-ng + assert z.nbytes_stored in (515, 485) class TestArrayNoCache(TestArray): @@ -2394,13 +2441,15 @@ def create_store(self): return store def expected(self): - return [ - "ab753fc81df0878589535ca9bad2816ba88d91bc", - "c16261446f9436b1e9f962e57ce3e8f6074abe8a", - "c2ef3b2fb2bc9dcace99cd6dad1a7b66cc1ea058", - "6e52f95ac15b164a8e96843a230fcee0e610729b", - "091fa99bc60706095c9ce30b56ce2503e0223f56", - ] + return ( + [ + "ab753fc81df0878589535ca9bad2816ba88d91bc", + "c16261446f9436b1e9f962e57ce3e8f6074abe8a", + "c2ef3b2fb2bc9dcace99cd6dad1a7b66cc1ea058", + "6e52f95ac15b164a8e96843a230fcee0e610729b", + "091fa99bc60706095c9ce30b56ce2503e0223f56", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @@ -2426,16 +2475,21 @@ def create_store(self): return store def expected(self): - return [ - "ab753fc81df0878589535ca9bad2816ba88d91bc", - "c16261446f9436b1e9f962e57ce3e8f6074abe8a", - "c2ef3b2fb2bc9dcace99cd6dad1a7b66cc1ea058", - "6e52f95ac15b164a8e96843a230fcee0e610729b", - "091fa99bc60706095c9ce30b56ce2503e0223f56", - ] + return ( + [ + "ab753fc81df0878589535ca9bad2816ba88d91bc", + "c16261446f9436b1e9f962e57ce3e8f6074abe8a", + "c2ef3b2fb2bc9dcace99cd6dad1a7b66cc1ea058", + "6e52f95ac15b164a8e96843a230fcee0e610729b", + "091fa99bc60706095c9ce30b56ce2503e0223f56", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") +@pytest.mark.filterwarnings( + "ignore:.*Support for partial decompression will be removed in a future version.*" +) class TestArrayWithFSStorePartialRead(TestArray): compressor = Blosc(blocksize=256) partial_decompress = True @@ -2447,13 +2501,15 @@ def create_store(self): return store def expected(self): - return [ - "dd7577d645c38767cf6f6d1ef8fd64002883a014", - "aa0de9892cf1ed3cda529efbf3233720b84489b7", - "e6191c44cf958576c29c41cef0f55b028a4dbdff", - "88adeeabb819feecccadf50152293dbb42f9107e", - "1426e084427f9920e29c9ec81b663d1005849455", - ] + return ( + [ + "dd7577d645c38767cf6f6d1ef8fd64002883a014", + "aa0de9892cf1ed3cda529efbf3233720b84489b7", + "e6191c44cf958576c29c41cef0f55b028a4dbdff", + "88adeeabb819feecccadf50152293dbb42f9107e", + "1426e084427f9920e29c9ec81b663d1005849455", + ], + ) def test_non_cont(self): z = self.create_array(shape=(500, 500, 500), chunks=(50, 50, 50), dtype=" DBMStoreV3: - import bsddb3 - - path = mktemp(suffix=".dbm") - atexit.register(os.remove, path) - store = DBMStoreV3(path, flag="n", open=bsddb3.btopen) - return store - - def test_nbytes_stored(self): - pass # not implemented - - @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") @pytest.mark.skipif(have_lmdb is False, reason="needs lmdb") class TestArrayWithLMDBStoreV3(TestArrayV3): @@ -2933,13 +2987,15 @@ def create_store(self): return store def expected(self): - return [ - "1509abec4285494b61cd3e8d21f44adc3cf8ddf6", - "7cfb82ec88f7ecb7ab20ae3cb169736bc76332b8", - "b663857bb89a8ab648390454954a9cdd453aa24b", - "21e90fa927d09cbaf0e3b773130e2dc05d18ff9b", - "e8c1fdd18b5c2ee050b59d0c8c95d07db642459c", - ] + return ( + [ + "1509abec4285494b61cd3e8d21f44adc3cf8ddf6", + "7cfb82ec88f7ecb7ab20ae3cb169736bc76332b8", + "b663857bb89a8ab648390454954a9cdd453aa24b", + "21e90fa927d09cbaf0e3b773130e2dc05d18ff9b", + "e8c1fdd18b5c2ee050b59d0c8c95d07db642459c", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @@ -2963,33 +3019,43 @@ def create_store(self): return store def expected(self): - return [ - "1509abec4285494b61cd3e8d21f44adc3cf8ddf6", - "7cfb82ec88f7ecb7ab20ae3cb169736bc76332b8", - "b663857bb89a8ab648390454954a9cdd453aa24b", - "21e90fa927d09cbaf0e3b773130e2dc05d18ff9b", - "e8c1fdd18b5c2ee050b59d0c8c95d07db642459c", - ] + return ( + [ + "1509abec4285494b61cd3e8d21f44adc3cf8ddf6", + "7cfb82ec88f7ecb7ab20ae3cb169736bc76332b8", + "b663857bb89a8ab648390454954a9cdd453aa24b", + "21e90fa927d09cbaf0e3b773130e2dc05d18ff9b", + "e8c1fdd18b5c2ee050b59d0c8c95d07db642459c", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") +@pytest.mark.filterwarnings( + "ignore:.*Support for partial decompression will be removed in a future version.*" +) class TestArrayWithFSStoreV3PartialRead(TestArrayWithFSStoreV3): partial_decompress = True def expected(self): - return [ - "1509abec4285494b61cd3e8d21f44adc3cf8ddf6", - "7cfb82ec88f7ecb7ab20ae3cb169736bc76332b8", - "b663857bb89a8ab648390454954a9cdd453aa24b", - "21e90fa927d09cbaf0e3b773130e2dc05d18ff9b", - "e8c1fdd18b5c2ee050b59d0c8c95d07db642459c", - ] + return ( + [ + "1509abec4285494b61cd3e8d21f44adc3cf8ddf6", + "7cfb82ec88f7ecb7ab20ae3cb169736bc76332b8", + "b663857bb89a8ab648390454954a9cdd453aa24b", + "21e90fa927d09cbaf0e3b773130e2dc05d18ff9b", + "e8c1fdd18b5c2ee050b59d0c8c95d07db642459c", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") @pytest.mark.skipif(not v3_sharding_available, reason="sharding is disabled") +@pytest.mark.filterwarnings( + "ignore:.*Support for partial decompression will be removed in a future version.*" +) class TestArrayWithFSStoreV3PartialReadUncompressedSharded(TestArrayWithFSStoreV3): partial_decompress = True compressor = None @@ -3015,13 +3081,15 @@ def test_supports_efficient_get_set_partial_values(self): assert not z.chunk_store.supports_efficient_set_partial_values() def expected(self): - return [ - "90109fc2a4e17efbcb447003ea1c08828b91f71e", - "2b73519f7260dba3ddce0d2b70041888856fec6b", - "bca5798be2ed71d444f3045b05432d937682b7dd", - "9ff1084501e28520e577662a6e3073f1116c76a2", - "882a97cad42417f90f111d0cb916a21579650467", - ] + return ( + [ + "90109fc2a4e17efbcb447003ea1c08828b91f71e", + "2b73519f7260dba3ddce0d2b70041888856fec6b", + "bca5798be2ed71d444f3045b05432d937682b7dd", + "9ff1084501e28520e577662a6e3073f1116c76a2", + "882a97cad42417f90f111d0cb916a21579650467", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @@ -3030,13 +3098,15 @@ class TestArrayWithFSStoreV3Nested(TestArrayWithFSStoreV3): dimension_separator = "/" def expected(self): - return [ - "1509abec4285494b61cd3e8d21f44adc3cf8ddf6", - "7cfb82ec88f7ecb7ab20ae3cb169736bc76332b8", - "b663857bb89a8ab648390454954a9cdd453aa24b", - "21e90fa927d09cbaf0e3b773130e2dc05d18ff9b", - "e8c1fdd18b5c2ee050b59d0c8c95d07db642459c", - ] + return ( + [ + "1509abec4285494b61cd3e8d21f44adc3cf8ddf6", + "7cfb82ec88f7ecb7ab20ae3cb169736bc76332b8", + "b663857bb89a8ab648390454954a9cdd453aa24b", + "21e90fa927d09cbaf0e3b773130e2dc05d18ff9b", + "e8c1fdd18b5c2ee050b59d0c8c95d07db642459c", + ], + ) @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") @@ -3045,13 +3115,15 @@ class TestArrayWithFSStoreV3NestedPartialRead(TestArrayWithFSStoreV3): dimension_separator = "/" def expected(self): - return [ - "1509abec4285494b61cd3e8d21f44adc3cf8ddf6", - "7cfb82ec88f7ecb7ab20ae3cb169736bc76332b8", - "b663857bb89a8ab648390454954a9cdd453aa24b", - "21e90fa927d09cbaf0e3b773130e2dc05d18ff9b", - "e8c1fdd18b5c2ee050b59d0c8c95d07db642459c", - ] + return ( + [ + "1509abec4285494b61cd3e8d21f44adc3cf8ddf6", + "7cfb82ec88f7ecb7ab20ae3cb169736bc76332b8", + "b663857bb89a8ab648390454954a9cdd453aa24b", + "21e90fa927d09cbaf0e3b773130e2dc05d18ff9b", + "e8c1fdd18b5c2ee050b59d0c8c95d07db642459c", + ], + ) @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") @@ -3062,13 +3134,15 @@ def create_storage_transformers(self, shape) -> Tuple[Any]: ) def expected(self): - return [ - "3fb9a4f8233b09ad02067b6b7fc9fd5caa405c7d", - "89c8eb364beb84919fc9153d2c1ed2696274ec18", - "73307055c3aec095dd1232c38d793ef82a06bd97", - "6152c09255a5efa43b1a115546e35affa00c138c", - "2f8802fc391f67f713302e84fad4fd8f1366d6c2", - ] + return ( + [ + "3fb9a4f8233b09ad02067b6b7fc9fd5caa405c7d", + "89c8eb364beb84919fc9153d2c1ed2696274ec18", + "73307055c3aec095dd1232c38d793ef82a06bd97", + "6152c09255a5efa43b1a115546e35affa00c138c", + "2f8802fc391f67f713302e84fad4fd8f1366d6c2", + ], + ) @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") @@ -3108,13 +3182,15 @@ def test_supports_efficient_get_set_partial_values(self): assert not z.chunk_store.supports_efficient_set_partial_values() def expected(self): - return [ - "90109fc2a4e17efbcb447003ea1c08828b91f71e", - "2b73519f7260dba3ddce0d2b70041888856fec6b", - "bca5798be2ed71d444f3045b05432d937682b7dd", - "9ff1084501e28520e577662a6e3073f1116c76a2", - "882a97cad42417f90f111d0cb916a21579650467", - ] + return ( + [ + "90109fc2a4e17efbcb447003ea1c08828b91f71e", + "2b73519f7260dba3ddce0d2b70041888856fec6b", + "bca5798be2ed71d444f3045b05432d937682b7dd", + "9ff1084501e28520e577662a6e3073f1116c76a2", + "882a97cad42417f90f111d0cb916a21579650467", + ], + ) @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index 8e586abfff..3778141356 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -2,6 +2,7 @@ import os.path import shutil import warnings +import numbers import numpy as np import pytest @@ -762,7 +763,13 @@ def test_create_with_storage_transformers(at_root): ) def test_shape_chunk_ints(init_shape, init_chunks, shape, chunks): g = open_group() - array = g.create_dataset("ds", shape=init_shape, chunks=init_chunks, dtype=np.uint8) + if not isinstance(init_shape[0], numbers.Integral) or not isinstance( + init_chunks[0], numbers.Integral + ): + with pytest.warns(UserWarning): + array = g.create_dataset("ds", shape=init_shape, chunks=init_chunks, dtype=np.uint8) + else: + array = g.create_dataset("ds", shape=init_shape, chunks=init_chunks, dtype=np.uint8) assert all( isinstance(s, int) for s in array.shape diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 161e1eb813..8a03616637 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -1439,27 +1439,6 @@ def create_store(): return store, None -class TestGroupWithDBMStoreBerkeleyDB(TestGroup): - @staticmethod - def create_store(): - bsddb3 = pytest.importorskip("bsddb3") - path = mktemp(suffix=".dbm") - atexit.register(os.remove, path) - store = DBMStore(path, flag="n", open=bsddb3.btopen) - return store, None - - -@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") -class TestGroupV3WithDBMStoreBerkeleyDB(TestGroupWithDBMStoreBerkeleyDB, TestGroupV3): - @staticmethod - def create_store(): - bsddb3 = pytest.importorskip("bsddb3") - path = mktemp(suffix=".dbm") - atexit.register(os.remove, path) - store = DBMStoreV3(path, flag="n", open=bsddb3.btopen) - return store, None - - class TestGroupWithLMDBStore(TestGroup): @staticmethod def create_store(): diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index da690f5959..d72718d77a 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1932,15 +1932,6 @@ def create_store(self, **kwargs): return store # pragma: no cover -class TestDBMStoreBerkeleyDB(TestDBMStore): - def create_store(self, **kwargs): - bsddb3 = pytest.importorskip("bsddb3") - path = mktemp(suffix=".dbm") - atexit.register(os.remove, path) - store = DBMStore(path, flag="n", open=bsddb3.btopen, write_lock=False, **kwargs) - return store - - class TestLMDBStore(StoreTests): def create_store(self, **kwargs): pytest.importorskip("lmdb") diff --git a/zarr/tests/test_storage_v3.py b/zarr/tests/test_storage_v3.py index e8675786e0..47e19d911b 100644 --- a/zarr/tests/test_storage_v3.py +++ b/zarr/tests/test_storage_v3.py @@ -53,7 +53,6 @@ from .test_storage import TestABSStore as _TestABSStore from .test_storage import TestConsolidatedMetadataStore as _TestConsolidatedMetadataStore from .test_storage import TestDBMStore as _TestDBMStore -from .test_storage import TestDBMStoreBerkeleyDB as _TestDBMStoreBerkeleyDB from .test_storage import TestDBMStoreDumb as _TestDBMStoreDumb from .test_storage import TestDBMStoreGnu as _TestDBMStoreGnu from .test_storage import TestDBMStoreNDBM as _TestDBMStoreNDBM @@ -465,15 +464,6 @@ def create_store(self, **kwargs): return store # pragma: no cover -class TestDBMStoreV3BerkeleyDB(_TestDBMStoreBerkeleyDB, StoreV3Tests): - def create_store(self, **kwargs): - bsddb3 = pytest.importorskip("bsddb3") - path = mktemp(suffix=".dbm") - atexit.register(os.remove, path) - store = DBMStoreV3(path, flag="n", open=bsddb3.btopen, write_lock=False, **kwargs) - return store - - class TestLMDBStoreV3(_TestLMDBStore, StoreV3Tests): def create_store(self, **kwargs): pytest.importorskip("lmdb") diff --git a/zarr/tests/test_util.py b/zarr/tests/test_util.py index d908c7b2d7..2b71566300 100644 --- a/zarr/tests/test_util.py +++ b/zarr/tests/test_util.py @@ -44,7 +44,8 @@ def test_normalize_shape(): with pytest.raises(TypeError): normalize_shape(None) with pytest.raises(ValueError): - normalize_shape("foo") + with pytest.warns(UserWarning): + normalize_shape("foo") def test_normalize_chunks(): diff --git a/zarr/util.py b/zarr/util.py index 8a96f92c24..2c9b0f616b 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -18,6 +18,7 @@ Iterable, cast, ) +import warnings import numpy as np from asciitree import BoxStyle, LeftAligned @@ -88,6 +89,8 @@ def normalize_shape(shape: Union[int, Tuple[int, ...], None]) -> Tuple[int, ...] # normalize shape = cast(Tuple[int, ...], shape) + if not all(isinstance(s, numbers.Integral) for s in shape): + warnings.warn("shape contains non-integer value(s)", UserWarning, stacklevel=2) shape = tuple(int(s) for s in shape) return shape @@ -176,6 +179,9 @@ def normalize_chunks(chunks: Any, shape: Tuple[int, ...], typesize: int) -> Tupl if -1 in chunks or None in chunks: chunks = tuple(s if c == -1 or c is None else int(c) for s, c in zip(shape, chunks)) + if not all(isinstance(c, numbers.Integral) for c in chunks): + warnings.warn("chunks contains non-integer value(s)", UserWarning, stacklevel=2) + chunks = tuple(int(c) for c in chunks) return chunks pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy