Skip to content

Commit 19900e1

Browse files
maxrjonesd-v-bdstansbyTomNicholas
committed
Create read only copy if needed when opening a store path (#3156)
* Create read only copy if needed when opening a store path * Add ValueError to Raises section * Update expected warning * Update src/zarr/storage/_common.py Co-authored-by: Davis Bennett <davis.v.bennett@gmail.com> * Use ANY_ACCESS_MODE * Update src/zarr/storage/_common.py Co-authored-by: David Stansby <dstansby@gmail.com> * Update src/zarr/storage/_common.py Co-authored-by: David Stansby <dstansby@gmail.com> * Update changes * Try using get_args on definition * Revert "Try using get_args on definition" This reverts commit 7ad760f. * Add test * Remove warning * Apply suggestion for try; except shortening Co-authored-by: Tom Nicholas <tom@earthmover.io> * Improve code coverage --------- Co-authored-by: Davis Bennett <davis.v.bennett@gmail.com> Co-authored-by: David Stansby <dstansby@gmail.com> Co-authored-by: Tom Nicholas <tom@earthmover.io> (cherry picked from commit 5731c6c)
1 parent 04a4c5f commit 19900e1

File tree

7 files changed

+92
-36
lines changed

7 files changed

+92
-36
lines changed

changes/3068.bugfix.rst

Lines changed: 0 additions & 1 deletion
This file was deleted.

changes/3156.bugfix.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Trying to open a StorePath/Array with ``mode='r'`` when the store is not read-only creates a read-only copy of the store.

src/zarr/core/common.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from typing import (
1111
TYPE_CHECKING,
1212
Any,
13+
Final,
1314
Literal,
1415
TypeVar,
1516
cast,
@@ -40,6 +41,7 @@
4041
JSON = str | int | float | Mapping[str, "JSON"] | Sequence["JSON"] | None
4142
MemoryOrder = Literal["C", "F"]
4243
AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"]
44+
ANY_ACCESS_MODE: Final = "r", "r+", "a", "w", "w-"
4345
DimensionNames = Iterable[str | None] | None
4446

4547

src/zarr/storage/_common.py

Lines changed: 55 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,14 @@
77

88
from zarr.abc.store import ByteRequest, Store
99
from zarr.core.buffer import Buffer, default_buffer_prototype
10-
from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZGROUP_JSON, AccessModeLiteral, ZarrFormat
10+
from zarr.core.common import (
11+
ANY_ACCESS_MODE,
12+
ZARR_JSON,
13+
ZARRAY_JSON,
14+
ZGROUP_JSON,
15+
AccessModeLiteral,
16+
ZarrFormat,
17+
)
1118
from zarr.errors import ContainsArrayAndGroupError, ContainsArrayError, ContainsGroupError
1219
from zarr.storage._local import LocalStore
1320
from zarr.storage._memory import MemoryStore
@@ -54,59 +61,82 @@ def __init__(self, store: Store, path: str = "") -> None:
5461
def read_only(self) -> bool:
5562
return self.store.read_only
5663

64+
@classmethod
65+
async def _create_open_instance(cls, store: Store, path: str) -> Self:
66+
"""Helper to create and return a StorePath instance."""
67+
await store._ensure_open()
68+
return cls(store, path)
69+
5770
@classmethod
5871
async def open(cls, store: Store, path: str, mode: AccessModeLiteral | None = None) -> Self:
5972
"""
6073
Open StorePath based on the provided mode.
6174
62-
* If the mode is 'w-' and the StorePath contains keys, raise a FileExistsError.
63-
* If the mode is 'w', delete all keys nested within the StorePath
64-
* If the mode is 'a', 'r', or 'r+', do nothing
75+
* If the mode is None, return an opened version of the store with no changes.
76+
* If the mode is 'r+', 'w-', 'w', or 'a' and the store is read-only, raise a ValueError.
77+
* If the mode is 'r' and the store is not read-only, return a copy of the store with read_only set to True.
78+
* If the mode is 'w-' and the store is not read-only and the StorePath contains keys, raise a FileExistsError.
79+
* If the mode is 'w' and the store is not read-only, delete all keys nested within the StorePath.
6580
6681
Parameters
6782
----------
6883
mode : AccessModeLiteral
6984
The mode to use when initializing the store path.
7085
86+
The accepted values are:
87+
88+
- ``'r'``: read only (must exist)
89+
- ``'r+'``: read/write (must exist)
90+
- ``'a'``: read/write (create if doesn't exist)
91+
- ``'w'``: read/write (overwrite if exists)
92+
- ``'w-'``: read/write (create if doesn't exist).
93+
7194
Raises
7295
------
7396
FileExistsError
7497
If the mode is 'w-' and the store path already exists.
7598
ValueError
7699
If the mode is not "r" and the store is read-only, or
77-
if the mode is "r" and the store is not read-only.
78100
"""
79101

80-
await store._ensure_open()
81-
self = cls(store, path)
82-
83102
# fastpath if mode is None
84103
if mode is None:
85-
return self
104+
return await cls._create_open_instance(store, path)
86105

87-
if store.read_only and mode != "r":
88-
raise ValueError(f"Store is read-only but mode is '{mode}'")
89-
if not store.read_only and mode == "r":
90-
raise ValueError(f"Store is not read-only but mode is '{mode}'")
106+
if mode not in ANY_ACCESS_MODE:
107+
raise ValueError(f"Invalid mode: {mode}, expected one of {ANY_ACCESS_MODE}")
91108

109+
if store.read_only:
110+
# Don't allow write operations on a read-only store
111+
if mode != "r":
112+
raise ValueError(
113+
f"Store is read-only but mode is {mode!r}. Create a writable store or use 'r' mode."
114+
)
115+
self = await cls._create_open_instance(store, path)
116+
elif mode == "r":
117+
# Create read-only copy for read mode on writable store
118+
try:
119+
read_only_store = store.with_read_only(True)
120+
except NotImplementedError as e:
121+
raise ValueError(
122+
"Store is not read-only but mode is 'r'. Unable to create a read-only copy of the store. "
123+
"Please use a read-only store or a storage class that implements .with_read_only()."
124+
) from e
125+
self = await cls._create_open_instance(read_only_store, path)
126+
else:
127+
# writable store and writable mode
128+
self = await cls._create_open_instance(store, path)
129+
130+
# Handle mode-specific operations
92131
match mode:
93132
case "w-":
94133
if not await self.is_empty():
95-
msg = (
96-
f"{self} is not empty, but `mode` is set to 'w-'."
97-
"Either remove the existing objects in storage,"
98-
"or set `mode` to a value that handles pre-existing objects"
99-
"in storage, like `a` or `w`."
134+
raise FileExistsError(
135+
f"Cannot create '{path}' with mode 'w-' because it already contains data. "
136+
f"Use mode 'w' to overwrite or 'a' to append."
100137
)
101-
raise FileExistsError(msg)
102138
case "w":
103139
await self.delete_dir()
104-
case "a" | "r" | "r+":
105-
# No init action
106-
pass
107-
case _:
108-
raise ValueError(f"Invalid mode: {mode}")
109-
110140
return self
111141

112142
async def get(

tests/test_api.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1318,7 +1318,7 @@ def test_no_overwrite_open(tmp_path: Path, open_func: Callable, mode: str) -> No
13181318
existing_fpath = add_empty_file(tmp_path)
13191319

13201320
assert existing_fpath.exists()
1321-
with contextlib.suppress(FileExistsError, FileNotFoundError, ValueError):
1321+
with contextlib.suppress(FileExistsError, FileNotFoundError, UserWarning):
13221322
open_func(store=store, mode=mode)
13231323
if mode == "w":
13241324
assert not existing_fpath.exists()

tests/test_common.py

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,36 @@
11
from __future__ import annotations
22

3-
from typing import TYPE_CHECKING
4-
5-
if TYPE_CHECKING:
6-
from collections.abc import Iterable
7-
from typing import Any, Literal
3+
from typing import TYPE_CHECKING, get_args
84

95
import numpy as np
106
import pytest
117

12-
from zarr.core.common import parse_name, parse_shapelike, product
8+
from zarr.core.common import (
9+
ANY_ACCESS_MODE,
10+
AccessModeLiteral,
11+
parse_name,
12+
parse_shapelike,
13+
product,
14+
)
1315
from zarr.core.config import parse_indexing_order
1416

17+
if TYPE_CHECKING:
18+
from collections.abc import Iterable
19+
from typing import Any, Literal
20+
1521

1622
@pytest.mark.parametrize("data", [(0, 0, 0, 0), (1, 3, 4, 5, 6), (2, 4)])
1723
def test_product(data: tuple[int, ...]) -> None:
1824
assert product(data) == np.prod(data)
1925

2026

27+
def test_access_modes() -> None:
28+
"""
29+
Test that the access modes type and variable for run-time checking are equivalent.
30+
"""
31+
assert set(ANY_ACCESS_MODE) == set(get_args(AccessModeLiteral))
32+
33+
2134
# todo: test
2235
def test_concurrent_map() -> None: ...
2336

tests/test_store/test_core.py

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import zarr
88
from zarr import Group
99
from zarr.core.common import AccessModeLiteral, ZarrFormat
10-
from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath
10+
from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath, ZipStore
1111
from zarr.storage._common import contains_array, contains_group, make_store_path
1212
from zarr.storage._utils import (
1313
_join_paths,
@@ -263,8 +263,19 @@ def test_relativize_path_invalid() -> None:
263263
_relativize_path(path="a/b/c", prefix="b")
264264

265265

266-
def test_invalid_open_mode() -> None:
266+
def test_different_open_mode(tmp_path: LEGACY_PATH) -> None:
267+
# Test with a store that implements .with_read_only()
267268
store = MemoryStore()
268269
zarr.create((100,), store=store, zarr_format=2, path="a")
269-
with pytest.raises(ValueError, match="Store is not read-only but mode is 'r'"):
270+
arr = zarr.open_array(store=store, path="a", zarr_format=2, mode="r")
271+
assert arr.store.read_only
272+
273+
# Test with a store that doesn't implement .with_read_only()
274+
zarr_path = tmp_path / "foo.zarr"
275+
store = ZipStore(zarr_path, mode="w")
276+
zarr.create((100,), store=store, zarr_format=2, path="a")
277+
with pytest.raises(
278+
ValueError,
279+
match="Store is not read-only but mode is 'r'. Unable to create a read-only copy of the store. Please use a read-only store or a storage class that implements .with_read_only().",
280+
):
270281
zarr.open_array(store=store, path="a", zarr_format=2, mode="r")

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy