docs: document filters and blobio

Author: pmrowla

docs/filters.rst

@@ -0,0 +1,63 @@
+**********************************************************************
+Filters
+**********************************************************************
+
+pygit2 supports defining and registering libgit2 blob filters implemented
+in Python.
+
+The Filter type
+===============
+
+.. autoclass:: pygit2.Filter
+   :members:
+
+.. autoclass:: pygit2.FilterSource
+
+Registering filters
+===================
+
+.. autofunction:: pygit2.filter_register
+.. autofunction:: pygit2.filter_unregister
+
+Example
+=======
+
+The following example is a simple Python implementation of a filter which
+enforces that blobs are stored with unix LF line-endings in the ODB, and
+checked out with line-endings in accordance with the .gitattributes ``eol``
+setting.
+
+.. code-block:: python
+
+    class CRLFFilter(pygit2.Filter):
+        attributes = "text eol=*"
+
+        def __init__(self):
+            super().__init__()
+            self.linesep = b'\r\n' if os.name == 'nt' else b'\n'
+            self.buffer = io.BytesIO()
+
+        def check(self, src, attr_values):
+            if src.mode == GIT_FILTER_SMUDGE:
+                # attr_values contains the values of the 'text' and 'eol'
+                # attributes in that order (as they are defined in
+                # CRLFFilter.attributes
+                eol = attr_values[1]
+
+                if eol == 'crlf':
+                    self.linesep = b'\r\n'
+                elif eol = 'lf':
+                    self.linesep = b'\n'
+            else:  # src.mode == GIT_FILTER_CLEAN
+                # always use LF line-endings when writing to the ODB
+                self.linesep = b'\n'
+
+        def write(data, src, write_next):
+            # buffer input data in case line-ending sequences span chunk boundaries
+            self.buffer.write(data)
+
+        def close(self, write_next):
+            # apply line-ending conversion to our buffered input and write all
+            # of our output data
+            self.buffer.seek(0)
+            write_next(self.linesep.join(self.buffer.read().splitlines()))

docs/index.rst

@@ -68,6 +68,7 @@ Table of Contents
    config
    diff
    features
+   filters
    index_file
    mailmap
    merge

docs/objects.rst

@@ -123,6 +123,20 @@ creating the blob object:
 .. autofunction:: pygit2.hash
 .. autofunction:: pygit2.hashfile
 
+Streaming blob content
+----------------------
+
+`pygit2.Blob.data` and `pygit2.Blob.read_raw()` read the full contents of the
+blob into memory and return Python ``bytes``. They also return the raw contents
+of the blob, and do not apply any filters which would be applied upon checkout
+to the working directory.
+
+Raw and filtered blob data can be accessed as a Python Binary I/O stream
+(i.e. a file-like object):
+
+.. autoclass:: pygit2.BlobIO
+   :members:
+
 
 Trees
 =================

pygit2/blob.py

@@ -100,6 +100,27 @@ class BlobIO(io.BufferedReader, AbstractContextManager):
 
     Supports reading both raw and filtered blob content.
     Implements io.BufferedReader.
+
+    Example:
+
+        >>> with BlobIO(blob) as f:
+        ...     while True:
+        ...         # Read blob data in 1KB chunks until EOF is reached
+        ...         chunk = f.read(1024)
+        ...         if not chunk:
+        ...             break
+
+    By default, `BlobIO` will stream the raw contents of the blob, but it
+    can also be used to stream filtered content (i.e. to read the content
+    after applying filters which would be used when checking out the blob
+    to the working directory).
+
+    Example:
+
+        >>> with BlobIO(blob, as_path='my_file.ext') as f:
+        ...     # Read the filtered content which would be returned upon
+        ...     # running 'git checkout -- my_file.txt'
+        ...     filtered_data = f.read()
     """
 
     def __init__(

pygit2/filter.py

@@ -32,15 +32,19 @@ class Filter:
     """
     Base filter class to be used with libgit2 filters.
 
+    Inherit from this class and override the `check()`, `write()` and `close()`
+    methods to define a filter which can then be registered via
+    `pygit2.filter_register()`.
+
     A new Filter instance will be instantiated for each stream which needs to
     be filtered. For each stream, filter methods will be called in this order:
 
         - `check()`
         - `write()` (may be called multiple times)
         - `close()`
 
-    Output data should be written to the next filter in the chain during
-    `write()` and `close()` via the `write_next` method. All output data
+    Filtered output data should be written to the next filter in the chain
+    during `write()` and `close()` via the `write_next` method. All output data
     must be written to the next filter before returning from `close()`.
 
     If a filter is dependent on reading the complete input data stream, the
@@ -61,6 +65,13 @@ def check(self, src: FilterSource, attr_values: List[str]):
         `check` will be called once per stream.
 
         If `Passthrough` is raised, the filter will not be applied.
+
+        Parameters:
+            src: The source of the filtered blob.
+            attr_values: The values of each attribute for the blob being
+                filtered. `attr_values` will be a sorted list containing
+                attributes in the order they were defined in
+                ``cls.attributes``.
         """
 
     def write(
@@ -74,7 +85,12 @@ def write(
 
         `write()` may be called multiple times per stream.
 
-        Output data should be written to `write_next` whenever it is available.
+        Parameters:
+            data: Input data.
+            src: The source of the filtered blob.
+            write_next: The ``write()`` method of the next filter in the chain.
+                Filtered output data should be written to `write_next` whenever
+                it is available.
         """
         write_next(data)
 
@@ -88,6 +104,8 @@ def close(
         `close()` will be called once per stream whenever all writes() to this
         stream have been completed.
 
-        Any remaining output data should be written to `write_next` before
-        returning.
+        Parameters:
+            write_next: The ``write()`` method of the next filter in the chain.
+                Any remaining filtered output data must be written to
+                `write_next` before returning.
         """