Merge branch 'main' into faster_delitems

Author: d-v-b

.github/workflows/releases.yml

@@ -64,7 +64,7 @@ jobs:
         with:
           name: releases
           path: dist
-      - uses: pypa/[email protected].7
+      - uses: pypa/[email protected].10
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.pre-commit-config.yaml

@@ -1,6 +1,7 @@
 ci:
   autoupdate_commit_msg: "chore: update pre-commit hooks"
   autofix_commit_msg: "style: pre-commit fixes"
+  autofix_prs: false
 default_stages: [commit, push]
 default_language_version:
   python: python3

.pyup.yml

@@ -13,6 +13,3 @@ requirements:
   - requirements_dev_optional.txt:
       pin: True
       update: all
-  - requirements_rtfd.txt:
-      pin: False
-      update: False

.readthedocs.yaml

@@ -10,6 +10,7 @@ sphinx:
 
 python:
   install:
-    - requirements: requirements_rtfd.txt
     - method: pip
       path: .
+      extra_requirements:
+        - docs

docs/contributing.rst

@@ -92,8 +92,8 @@ the repository, you can do something like the following::
     $ mkdir -p ~/pyenv/zarr-dev
     $ python -m venv ~/pyenv/zarr-dev
     $ source ~/pyenv/zarr-dev/bin/activate
-    $ pip install -r requirements_dev_minimal.txt -r requirements_dev_numpy.txt -r requirements_rtfd.txt
-    $ pip install -e .
+    $ pip install -r requirements_dev_minimal.txt -r requirements_dev_numpy.txt
+    $ pip install -e .[docs]
 
 To verify that your development environment is working, you can run the unit tests::
 

docs/release.rst

@@ -18,12 +18,23 @@ Release notes
 Unreleased
 ----------
 
+.. _release_2.16.1:
+
+2.16.1
+------
+
 Maintenance
 ~~~~~~~~~~~
 
 * Require ``setuptools_scm`` version ``1.5.4``\+
   By :user:`John A. Kirkham <jakirkham>` :issue:`1477`.
 
+* Add ``docs`` requirements to ``pyproject.toml``
+  By :user:`John A. Kirkham <jakirkham>` :issue:`1494`.
+
+* Fixed caching issue in ``LRUStoreCache``.
+  By :user:`Mads R. B. Kristensen <madsbk>` :issue:`1499`.
+
 .. _release_2.16.0:
 
 2.16.0

pyproject.toml

@@ -13,7 +13,7 @@ maintainers = [
 requires-python = ">=3.8"
 dependencies = [
     'asciitree',
-    'numpy>=1.20',
+    'numpy>=1.20,!=1.21.0',
     'fasteners',
     'numcodecs>=0.10.0',
 ]
@@ -43,6 +43,16 @@ jupyter = [
     'ipytree>=0.2.2',
     'ipywidgets>=8.0.0',
 ]
+docs = [
+    'sphinx',
+    'sphinx_design',
+    'sphinx-issues',
+    'sphinx-copybutton',
+    'sphinx-rtd-theme',
+    'pydata-sphinx-theme',
+    'numpydoc',
+    'numcodecs[msgpack]',
+]
 
 [project.urls]
 "Bug Tracker" = "https://github.com/zarr-developers/zarr-python/issues"

requirements_dev_optional.txt

@@ -3,12 +3,12 @@
 lmdb==1.4.1; sys_platform != 'win32'
 # optional library requirements for Jupyter
 ipytree==0.2.2
-ipywidgets==8.0.7
+ipywidgets==8.1.0
 # optional library requirements for services
 # don't let pyup change pinning for azure-storage-blob, need to pin to older
 # version to get compatibility with azure storage emulator on appveyor (FIXME)
 azure-storage-blob==12.16.0 # pyup: ignore
-redis==4.5.5
+redis==4.6.0
 types-redis
 types-setuptools
 pymongo==4.4.0

requirements_rtfd.txt

@@ -509,7 +509,7 @@ def __init__(self, store, max_size: int):
         self._max_size = max_size
         self._current_size = 0
         self._keys_cache = None
-        self._contains_cache = None
+        self._contains_cache = {}
         self._listdir_cache: Dict[Path, Any] = dict()
         self._values_cache: Dict[Path, Any] = OrderedDict()
         self._mutex = Lock()

zarr/_storage/v3.py

@@ -1360,7 +1360,8 @@ def group(
     synchronizer=None,
     path=None,
     *,
-    zarr_version=None
+    zarr_version=None,
+    meta_array=None
 ):
     """Create a group.
 
@@ -1382,6 +1383,11 @@ def group(
         Array synchronizer.
     path : string, optional
         Group path within store.
+    meta_array : array-like, optional
+        An array instance to use for determining arrays to create and return
+        to users. Use `numpy.empty(())` by default.
+
+        .. versionadded:: 2.16.1
 
     Returns
     -------
@@ -1432,6 +1438,7 @@ def group(
         synchronizer=synchronizer,
         path=path,
         zarr_version=zarr_version,
+        meta_array=meta_array,
     )
 
 

zarr/hierarchy.py

@@ -2403,7 +2403,7 @@ def __init__(self, store: StoreLike, max_size: int):
         self._max_size = max_size
         self._current_size = 0
         self._keys_cache = None
-        self._contains_cache = None
+        self._contains_cache: Dict[Any, Any] = {}
         self._listdir_cache: Dict[Path, Any] = dict()
         self._values_cache: Dict[Path, Any] = OrderedDict()
         self._mutex = Lock()
@@ -2444,9 +2444,9 @@ def __iter__(self):
 
     def __contains__(self, key):
         with self._mutex:
-            if self._contains_cache is None:
-                self._contains_cache = set(self._keys())
-            return key in self._contains_cache
+            if key not in self._contains_cache:
+                self._contains_cache[key] = key in self._store
+            return self._contains_cache[key]
 
     def clear(self):
         self._store.clear()
@@ -2516,7 +2516,7 @@ def invalidate_keys(self):
 
     def _invalidate_keys(self):
         self._keys_cache = None
-        self._contains_cache = None
+        self._contains_cache.clear()
         self._listdir_cache.clear()
 
     def _invalidate_value(self, key):

zarr/storage.py

@@ -9,7 +9,7 @@
 import zarr.codecs
 from zarr.core import Array
 from zarr.creation import array, empty, full, ones, open_array, zeros
-from zarr.hierarchy import open_group
+from zarr.hierarchy import open_group, group
 from zarr.storage import DirectoryStore, MemoryStore, Store, ZipStore
 
 
@@ -234,12 +234,13 @@ def test_full(module, compressor):
     assert np.all(np.isnan(z[:]))
 
 
+@pytest.mark.parametrize("group_create_function", [group, open_group])
 @pytest.mark.parametrize("module, compressor", param_module_and_compressor)
 @pytest.mark.parametrize("store_type", [None, DirectoryStore, MemoryStore, ZipStore])
-def test_group(tmp_path, module, compressor, store_type):
+def test_group(tmp_path, group_create_function, module, compressor, store_type):
     xp = ensure_module(module)
     store = init_store(tmp_path, store_type)
-    g = open_group(store, meta_array=xp.empty(()))
+    g = group_create_function(store, meta_array=xp.empty(()))
     g.ones("data", shape=(10, 11), dtype=int, compressor=compressor)
     a = g["data"]
     assert a.shape == (10, 11)

zarr/tests/test_meta_array.py

@@ -2196,7 +2196,10 @@ def test_cache_keys(self):
         assert keys == sorted(cache.keys())
         assert 1 == store.counter["keys"]
         assert foo_key in cache
-        assert 0 == store.counter["__contains__", foo_key]
+        assert 1 == store.counter["__contains__", foo_key]
+        # the next check for `foo_key` is cached
+        assert foo_key in cache
+        assert 1 == store.counter["__contains__", foo_key]
         assert keys == sorted(cache)
         assert 0 == store.counter["__iter__"]
         assert 1 == store.counter["keys"]
@@ -2215,23 +2218,23 @@ def test_cache_keys(self):
         keys = sorted(cache.keys())
         assert keys == [bar_key, baz_key, foo_key]
         assert 3 == store.counter["keys"]
-        assert 0 == store.counter["__contains__", foo_key]
+        assert 1 == store.counter["__contains__", foo_key]
         assert 0 == store.counter["__iter__"]
         cache.invalidate_keys()
         keys = sorted(cache)
         assert keys == [bar_key, baz_key, foo_key]
         assert 4 == store.counter["keys"]
-        assert 0 == store.counter["__contains__", foo_key]
+        assert 1 == store.counter["__contains__", foo_key]
         assert 0 == store.counter["__iter__"]
         cache.invalidate_keys()
         assert foo_key in cache
-        assert 5 == store.counter["keys"]
-        assert 0 == store.counter["__contains__", foo_key]
+        assert 4 == store.counter["keys"]
+        assert 2 == store.counter["__contains__", foo_key]
         assert 0 == store.counter["__iter__"]
 
         # check these would get counted if called directly
         assert foo_key in store
-        assert 1 == store.counter["__contains__", foo_key]
+        assert 3 == store.counter["__contains__", foo_key]
         assert keys == sorted(store)
         assert 1 == store.counter["__iter__"]