v4.2.1 Release

Author: khustup2

python/deeplake/__init__.py

@@ -16,7 +16,7 @@ def progress_bar(iterable, *args, **kwargs):
 from deeplake.ingestion import from_coco
 
 
-__version__ = "4.1.16"
+__version__ = "4.2.1"
 
 __all__ = [
     "__version__",
@@ -50,6 +50,7 @@ def progress_bar(iterable, *args, **kwargs):
     "DimensionsMismatch",
     "DtypeMismatch",
     "EmbeddingSizeMismatch",
+    "Executor",
     "ExpiredTokenError",
     "FormatNotSupportedError",
     "Future",
@@ -125,6 +126,7 @@ def progress_bar(iterable, *args, **kwargs):
     "UnsupportedPythonType",
     "UnsupportedSampleCompression",
     "Version",
+    "VersionNotFoundError",
     "WriteFailedError",
     "WrongChunkCompression",
     "WrongSampleCompression",
@@ -146,6 +148,7 @@ def progress_bar(iterable, *args, **kwargs):
     "open_async",
     "open_read_only",
     "open_read_only_async",
+    "prepare_query",
     "query",
     "query_async",
     "schemas",
@@ -239,7 +242,9 @@ def transfer_with_links(source, dest, links, column_names):
         iterable_cols = [col for col in column_names if col not in links]
         link_sample_info = {link: source[link]._links_info() for link in links}
         dest.set_creds_key(link_sample_info[links[0]]["key"])
-        pref_ds = source.query(f"SELECT {','.join(iterable_cols)}")
+        quoted_cols = ['"' + col + '"' for col in iterable_cols]
+        joined_cols = ",".join(quoted_cols)
+        pref_ds = source.query(f"SELECT {joined_cols}")
         dl = deeplake._deeplake._Prefetcher(pref_ds, raw_columns=set(get_raw_columns(source)))
 
         for counter, batch in enumerate(progress_bar(dl), start=1):
@@ -255,7 +260,7 @@ def transfer_with_links(source, dest, links, column_names):
                 commit_data(dest)
         commit_data(dest, "Final commit of linked data")
 
-    source_ds = deeplake.query(f'select * from "{src}"', token=token)
+    source_ds = deeplake.query(f'SELECT * FROM "{src}"', token=token)
     dest_ds = deeplake.like(source_ds, dst, dst_creds, token=token)
     commit_data(dest_ds, "Created dataset")
 

python/deeplake/__init__.pyi

@@ -38,6 +38,7 @@ __all__ = [
     "DimensionsMismatch",
     "DtypeMismatch",
     "EmbeddingSizeMismatch",
+    "Executor",
     "ExpiredTokenError",
     "FormatNotSupportedError",
     "Future",
@@ -113,6 +114,7 @@ __all__ = [
     "UnsupportedPythonType",
     "UnsupportedSampleCompression",
     "Version",
+    "VersionNotFoundError",
     "WriteFailedError",
     "WrongChunkCompression",
     "WrongSampleCompression",
@@ -134,6 +136,7 @@ __all__ = [
     "open_async",
     "open_read_only",
     "open_read_only_async",
+    "prepare_query",
     "query",
     "query_async",
     "schemas",
@@ -160,6 +163,9 @@ class Future:
         __await__() -> typing.Any:
             Enables using the Future in async/await syntax.
 
+        cancel() -> None:
+            Cancels the Future if it is still pending.
+
         is_completed() -> bool:
             Checks if the Future has resolved without blocking.
     <!-- test-context
@@ -274,6 +280,11 @@ class Future:
         """
         ...
 
+    def cancel(self) -> None:
+        """
+        Cancels the Future if it is still pending.
+        """
+
 class FutureVoid:
     """
     A Future representing a void async operation in ML pipelines.
@@ -291,6 +302,9 @@ class FutureVoid:
         is_completed() -> bool:
             Checks completion status without blocking.
 
+        cancel() -> None:
+            Cancels the Future if still pending.
+
     <!-- test-context
     ```python
     import deeplake
@@ -362,6 +376,11 @@ class FutureVoid:
         """
         ...
 
+    def cancel(self) -> None:
+        """
+        Cancels the Future if it is still pending.
+        """
+
     def is_completed(self) -> bool:
         """
         Checks if the operation has completed without blocking.
@@ -540,6 +559,38 @@ class Metadata(ReadOnlyMetadata):
         """
         ...
 
+def prepare_query(query: str, token: str | None = None, creds: dict[str, str] | None = None) -> Executor:
+    """
+    Prepares a TQL query for execution with optional authentication.
+
+    Args:
+        query: TQL query string to execute
+        token: Optional Activeloop authentication token
+        creds (dict, optional): Dictionary containing credentials used to access the dataset at the path.
+
+    Returns:
+        Executor: An executor object to run the query.
+
+    <!-- test-context
+    ```python
+    import deeplake
+    ds = deeplake.create("mem://parametriized")
+    ds.add_column("category", "text")
+    ds.append({"category": ["active", "inactive", "not sure"]})
+    ds.commit()
+    ```
+    -->
+
+    Examples:
+        Running a parametrized batch query:
+        ```python
+        ex = deeplake.prepare_query('SELECT * FROM "mem://parametriized" WHERE category = ?')
+        results = ex.run_batch([["active"], ["inactive"]])
+        assert len(results) == 2
+        ```
+    """
+    ...
+
 def query(query: str, token: str | None = None, creds: dict[str, str] | None = None) -> DatasetView:
     """
     Executes TQL queries optimized for ML data filtering and search.
@@ -1627,6 +1678,11 @@ class Row:
             or await the FutureVoid object in an asynchronous context.
         """
 
+    def to_dict(self) -> dict:
+        """
+        Converts the row to a dictionary.
+        """
+
     def __str__(self) -> str: ...
     @property
     def row_id(self) -> int:
@@ -1832,6 +1888,11 @@ class RowView:
             or use the Future in an `await` expression.
         """
 
+    def to_dict(self) -> dict:
+        """
+        Converts the row to a dictionary.
+        """
+
     def __str__(self) -> str: ...
     @property
     def row_id(self) -> int:
@@ -1942,6 +2003,35 @@ class DatasetView:
         """
         ...
 
+    def prepare_query(self, query: str) -> Executor:
+        """
+        Prepares a query for execution.
+
+        Parameters:
+            query: The query to prepare
+
+        Returns:
+            Executor: The prepared query
+
+        <!-- test-context
+        ```python
+        import deeplake
+        ds = deeplake.create("tmp://")
+        ds.add_column("category", "text")
+        ds.append({"category": ["active", "inactive", "not sure"]})
+        ```
+        -->
+
+        Examples:
+            ```python
+            executor = ds.prepare_query("select * where category == ?")
+            results = executor.run_batch([['active'], ['inactive'], ['not sure']])
+            for row in results:
+                print("Id is: ", row["category"])
+            ```
+        """
+        ...
+
     def query(self, query: str) -> DatasetView:
         """
         Executes the given TQL query against the dataset and return the results as a [deeplake.DatasetView][].
@@ -2675,6 +2765,26 @@ class Dataset(DatasetView):
         """
         ...
 
+    def refresh(
+        self
+    ) -> None:
+        """
+        Refreshes any new info from the dataset.
+
+        Similar to [deeplake.Dataset.open_read_only][] but the lightweight way.
+        """
+        ...
+
+    def refresh_async(
+        self
+    ) -> FutureVoid:
+        """
+        Asynchronously refreshes any new info from the dataset.
+
+        Similar to [deeplake.Dataset.open_read_only_async][] but the lightweight way.
+        """
+        ...
+
     @property
     def history(self) -> History:
         """
@@ -2816,6 +2926,26 @@ class ReadOnlyDataset(DatasetView):
         """
         ...
 
+    def refresh(
+        self
+    ) -> None:
+        """
+        Refreshes any new info from the dataset.
+
+        Similar to [deeplake.Dataset.open_read_only][] but the lightweight way.
+        """
+        ...
+
+    def refresh_async(
+        self
+    ) -> FutureVoid:
+        """
+        Asynchronously refreshes any new info from the dataset.
+
+        Similar to [deeplake.Dataset.open_read_only_async][] but the lightweight way.
+        """
+        ...
+
     def __getstate__(self) -> tuple:
         """Returns a dict that can be pickled and used to restore this dataset.
 
@@ -2830,6 +2960,14 @@ class ReadOnlyDataset(DatasetView):
             state (dict): The pickled state used to restore the dataset.
         """
 
+class Executor:
+    def get_query_string(self) -> str:
+        ...
+    def run_single(self) -> DatasetView:
+        ...
+    def run_batch(self, parameters: list = None) -> list:
+        ...
+
 class ExpiredTokenError(Exception):
     pass
 
@@ -3132,6 +3270,9 @@ class StorageNetworkConnectionError(Exception):
 class StorageInternalError(Exception):
     pass
 
+class VersionNotFoundError(Exception):
+    pass
+
 class WriteFailedError(Exception):
     pass
 
@@ -3703,4 +3844,4 @@ class TelemetryClient:
     Client for logging deeplake messages to telemetry.
     """
     endpoint: str
-    api_key: str
+    api_key: str

python/deeplake/storage.py

@@ -1,3 +1,3 @@
 from ._deeplake.types import *
 
-__all__ = ["Reader", "Writer", "ResourceMeta"]
+__all__ = ["Reader", "Writer", "ResourceMeta", "concurrency", "set_concurrency"]

python/deeplake/storage.pyi

@@ -7,7 +7,7 @@ import datetime
 import deeplake._deeplake.core
 import typing
 
-__all__ = ["Reader", "Writer", "ResourceMeta"]
+__all__ = ["Reader", "Writer", "ResourceMeta", "concurrency", "set_concurrency"]
 
 class Reader:
     def __getstate__(self) -> tuple: ...
@@ -59,3 +59,37 @@ class ResourceMeta:
     @property
     def size(self) -> int:
         ...
+
+def concurrency() -> int:
+    """
+    Returns the number of threads of storage readers and writers.
+
+    <!-- test-context
+    ```python
+    import deeplake
+    ```
+    -->
+
+    Examples:
+        ```python
+        deeplake.storage.concurrency()
+        ```
+    """
+    ...
+
+def set_concurrency(num_threads: int) -> None:
+    """
+    Sets the number of threads of storage readers and writers.
+
+    <!-- test-context
+    ```python
+    import deeplake
+    ```
+    -->
+
+    Examples:
+        ```python
+        deeplake.storage.set_concurrency(64)
+        ```
+    """
+

python/deeplake/tql.py

@@ -1,5 +1,7 @@
 from ._deeplake.tql import *
 
 __all__ = [
+    "get_max_num_parallel_queries",
     "register_function",
+    "set_max_num_parallel_queries",
 ]