Merge branch 'main' into f-strings

Author: joshmoore

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -27,7 +27,7 @@ body:
     attributes:
       label: Python Version
       description: Version of Python interpreter
-      placeholder: 3.8.5, 3.9, 3.10, etc.
+      placeholder: 3.9, 3.10, 3.11, etc.
     validations:
       required: true
   - type: input

.github/workflows/codeql-analysis.yml

@@ -42,7 +42,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -56,7 +56,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -69,4 +69,4 @@ jobs:
     #   ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/python-package.yml

@@ -15,13 +15,13 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
-        numpy_version: ['>=1.22.0', '==1.20.*']
+        python-version: ['3.9', '3.10', '3.11']
+        numpy_version: ['>=1.22.0', '==1.21.*']
         exclude:
           - python-version: '3.10'
-            numpy_version: '==1.20.*'
+            numpy_version: '==1.21.*'
           - python-version: '3.11'
-            numpy_version: '==1.20.*'
+            numpy_version: '==1.21.*'
     services:
       redis:
         image: redis

.github/workflows/releases.yml

@@ -16,10 +16,10 @@ jobs:
           submodules: true
           fetch-depth: 0
 
-      - uses: actions/setup-python@v4.7.1
+      - uses: actions/setup-python@v5.0.0
         name: Install Python
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install PyBuild
         run: |
@@ -36,7 +36,7 @@ jobs:
           else
             echo "All seem good"
           fi
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: releases
           path: dist
@@ -45,7 +45,7 @@ jobs:
     needs: [build_artifacts]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: releases
           path: dist
@@ -60,7 +60,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: releases
           path: dist

.github/workflows/windows-testing.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: True
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.10', '3.11']
     steps:
       - uses: actions/checkout@v4
         with:

.pre-commit-config.yaml

@@ -6,28 +6,25 @@ default_stages: [commit, push]
 default_language_version:
   python: python3
 repos:
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
+  - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.0.224'
+    rev: 'v0.2.1'
     hooks:
     - id: ruff
-      # Respect `exclude` and `extend-exclude` settings.
-      args: ["--force-exclude"]
   - repo: https://github.com/psf/black
-    rev: 23.10.1
+    rev: 24.2.0
     hooks:
     - id: black
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.5
+    rev: v2.2.6
     hooks:
       - id: codespell
-        args: ["-L", "ba,ihs,kake,nd,noe,nwo,te,fo,zar", "-S", "fixture"]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     - id: check-yaml
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.3.0
+    rev: v1.8.0
     hooks:
       - id: mypy
         files: zarr

TEAM.md

@@ -0,0 +1,26 @@
+## Active core-developers
+- @joshmoore (Josh Moore)
+- @jni (Juan Nunez-Iglesias)
+- @rabernat (Ryan Abernathey)
+- @jhamman (Joe Hamman)
+- @d-v-b (Davis Bennett)
+- @jakirkham (jakirkham)
+- @martindurant (Martin Durant)
+- @normanrz (Norman Rzepka)
+
+## Emeritus core-developers
+- @alimanfoo (Alistair Miles)
+- @shoyer (Stephan Hoyer)
+- @ryan-williams (Ryan Williams)
+- @jrbourbeau (James Bourbeau)
+- @mzjp2 (Zain Patel)
+- @grlee77 (Gregory Lee)
+
+## Former core-developers
+- @jeromekelleher (Jerome Kelleher)
+- @tjcrone (Tim Crone)
+- @funkey (Jan Funke)
+- @shikharsg
+- @Carreau (Matthias Bussonnier)
+- @dazzag24
+- @WardF (Ward Fisher)

docs/release.rst

@@ -18,6 +18,23 @@ Release notes
 Unreleased
 ----------
 
+* Change occurrences of % and format() to f-strings.
+  By :user:`Dimitri Papadopoulos Orfanos <DimitriPapadopoulos>` :issue:`1423`.
+
+.. _release_2.17.0:
+
+2.17.0
+------
+
+Enhancements
+~~~~~~~~~~~~
+
+* Added type hints to ``zarr.creation.create()``.
+  By :user:`David Stansby <dstansby>` :issue:`1536`.
+
+* Pyodide support: Don't require fasteners on Emscripten.
+  By :user:`Hood Chatham <hoodmane>` :issue:`1663`.
+
 Docs
 ~~~~
 
@@ -39,10 +56,27 @@ Docs
 * Minor tweak to advanced indexing tutorial examples.
   By :user:`Ross Barnowski <rossbar>` :issue:`1550`.
 
+* Automatically document array members using sphinx-automodapi.
+  By :user:`David Stansby <dstansby>` :issue:`1547`.
+
+* Add a markdown file documenting the current and former core-developer team.
+  By :user:`Joe Hamman <jhamman>` :issue:`1628`.
+
+* Add Norman Rzepka to core-dev team.
+  By :user:`Joe Hamman <jhamman>` :issue:`1630`.
+
+* Added section about accessing zip files that are on s3.
+  By :user:`Jeff Peck <jeffpeck10x>` :issue:`1613`.
+
+* Add V3 roadmap and design document.
+  By :user:`Joe Hamman <jhamman>` :issue:`1583`.
 
 Maintenance
 ~~~~~~~~~~~
 
+* Drop Python 3.8 and NumPy 1.20
+  By :user:`Josh Moore <joshmoore>`; :issue:`1557`.
+
 * Cache result of ``FSStore._fsspec_installed()``.
   By :user:`Janick Martinez Esturo <ph03>` :issue:`1581`.
 
@@ -64,8 +98,26 @@ Maintenance
 * Remove ``sphinx-rtd-theme`` dependency from ``pyproject.toml``.
   By :user:`Sanket Verma <MSanKeys963>` :issue:`1563`.
 
-* Change occurrences of % and format() to f-strings.
-  By :user:`Dimitri Papadopoulos Orfanos <DimitriPapadopoulos>` :issue:`1423`.
+* Remove ``CODE_OF_CONDUCT.md`` file from the Zarr-Python repository.
+  By :user:`Sanket Verma <MSanKeys963>` :issue:`1572`.
+
+* Bump version of black in pre-commit.
+  By :user:`David Stansby <dtstansby>` :issue:`1559`.
+
+* Use list comprehension where applicable.
+  By :user:`Dimitri Papadopoulos Orfanos <DimitriPapadopoulos>` :issue:`1555`.
+
+* Use format specification mini-language to format string.
+  By :user:`Dimitri Papadopoulos Orfanos <DimitriPapadopoulos>` :issue:`1558`.
+
+* Single startswith() call instead of multiple ones.
+  By :user:`Dimitri Papadopoulos Orfanos <DimitriPapadopoulos>` :issue:`1556`.
+
+* Move codespell options around.
+  By :user:`Dimitri Papadopoulos Orfanos <DimitriPapadopoulos>` :issue:`1196`.
+
+* Remove unused mypy ignore comments.
+  By :user:`David Stansby <dtstansby>` :issue:`1602`.
 
 .. _release_2.16.1:
 

docs/tutorial.rst

@@ -1000,6 +1000,31 @@ separately from Zarr.
 
 .. _tutorial_copy:
 
+Accessing Zip Files on S3
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The built-in `ZipStore` will only work with paths on the local file-system, however
+it is also possible to access ``.zarr.zip`` data on the cloud. Here is an example of
+accessing a zipped Zarr file on s3:
+
+    >>> s3_path = "s3://path/to/my.zarr.zip"
+    >>> 
+    >>> s3 = s3fs.S3FileSystem()
+    >>> f = s3.open(s3_path)
+    >>> fs = ZipFileSystem(f, mode="r")
+    >>> store = FSMap("", fs, check=False)
+    >>> 
+    >>> # cache is optional, but may be a good idea depending on the situation
+    >>> cache = zarr.storage.LRUStoreCache(store, max_size=2**28)
+    >>> z = zarr.group(store=cache)
+
+This store can also be generated with ``fsspec``'s handler chaining, like so:
+
+    >>> store = zarr.storage.FSStore(url=f"zip::{s3_path}",  mode="r")
+
+This can be especially useful if you have a very large ``.zarr.zip`` file on s3
+and only need to access a small portion of it.
+
 Consolidating metadata
 ~~~~~~~~~~~~~~~~~~~~~~
 

environment.yml

@@ -4,7 +4,7 @@ channels:
 dependencies:
   - wheel
   - numcodecs >= 0.6.4
-  - numpy >= 1.20
+  - numpy >= 1.21
   - pip
   - pip:
     - asciitree

pyproject.toml

@@ -10,11 +10,11 @@ readme = { file = "README.md", content-type = "text/markdown" }
 maintainers = [
     { name = "Alistair Miles", email = "[email protected]" }
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
     'asciitree',
-    'numpy>=1.20,!=1.21.0',
-    'fasteners',
+    'numpy>=1.21.1',
+    'fasteners; sys_platform != "emscripten"',
     'numcodecs>=0.10.0',
 ]
 dynamic = [
@@ -30,7 +30,6 @@ classifiers = [
     'Topic :: Software Development :: Libraries :: Python Modules',
     'Operating System :: Unix',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.8',
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
@@ -65,6 +64,7 @@ Homepage = "https://github.com/zarr-developers/zarr-python"
 exclude_lines = [
     "pragma: no cover",
     "pragma: ${PY_MAJOR_VERSION} no cover",
+    '.*\.\.\.' # Ignore "..." lines
 ]
 
 [tool.coverage.run]
@@ -119,9 +119,10 @@ exclude = '''
 '''
 
 [tool.mypy]
-python_version = "3.8"
 ignore_missing_imports = true
-follow_imports = "silent"
+warn_unused_configs = true
+warn_redundant_casts = true
+warn_unused_ignores = true
 
 [tool.pytest.ini_options]
 doctest_optionflags = [
@@ -137,3 +138,8 @@ filterwarnings = [
     "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning",
     "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning",
 ]
+
+
+[tool.codespell]
+ignore-words-list = "ba,ihs,kake,nd,noe,nwo,te,fo,zar"
+skip = 'fixture,.git'

requirements_dev_minimal.txt

@@ -5,4 +5,4 @@ numcodecs==0.12.1
 msgpack-python==0.5.6
 setuptools-scm==8.0.4
 # test requirements
-pytest==7.4.3
+pytest==7.4.4

requirements_dev_numpy.txt

@@ -1,4 +1,4 @@
 # Break this out into a separate file to allow testing against
 # different versions of numpy. This file should pin to the latest
 # numpy version.
-numpy==1.24.3
+numpy==1.26.1

requirements_dev_optional.txt

@@ -3,7 +3,7 @@
 lmdb==1.4.1; sys_platform != 'win32'
 # optional library requirements for Jupyter
 ipytree==0.2.2
-ipywidgets==8.1.0
+ipywidgets==8.1.1
 # optional library requirements for services
 # don't let pyup change pinning for azure-storage-blob, need to pin to older
 # version to get compatibility with azure storage emulator on appveyor (FIXME)
@@ -15,9 +15,9 @@ pymongo==4.6.1
 # optional test requirements
 coverage
 pytest-cov==4.1.0
-pytest-doctestplus==1.0.0
+pytest-doctestplus==1.1.0
 pytest-timeout==2.2.0
 h5py==3.10.0
-fsspec==2023.10.0
-s3fs==2023.10.0
-moto[server]>=4.0.8
+fsspec==2023.12.2
+s3fs==2023.12.2
+moto[server]>=5.0.1