Google Cloud Storage support using gcsfs

Author: bnaul

ci/appveyor-27.yaml

@@ -6,6 +6,7 @@ dependencies:
   - beautifulsoup4
   - bottleneck
   - dateutil
+  - gcsfs
   - html5lib
   - jinja2=2.8
   - lxml

ci/check_imports.py

@@ -5,6 +5,7 @@
 
 blacklist = {
     'bs4',
+    'gcsfs',
     'html5lib',
     'ipython',
     'jinja2'

ci/circle-36-locale_slow.yaml

@@ -5,6 +5,7 @@ channels:
 dependencies:
   - beautifulsoup4
   - cython
+  - gcsfs
   - html5lib
   - ipython
   - jinja2

ci/requirements-optional-conda.txt

@@ -3,6 +3,7 @@ blosc
 bottleneck
 fastparquet
 feather-format
+gcsfs
 html5lib
 ipython>=5.6.0
 ipykernel

ci/requirements-optional-pip.txt

@@ -5,6 +5,7 @@ blosc
 bottleneck
 fastparquet
 feather-format
+gcsfs
 html5lib
 ipython>=5.6.0
 ipykernel

ci/travis-27.yaml

@@ -9,6 +9,7 @@ dependencies:
   - fastparquet
   - feather-format
   - flake8=3.4.1
+  - gcsfs
   - html5lib
   - ipython
   - jemalloc=4.5.0.post

ci/travis-36.yaml

@@ -8,6 +8,7 @@ dependencies:
   - dask
   - fastparquet
   - feather-format
+  - gcsfs
   - geopandas
   - html5lib
   - ipython

doc/source/install.rst

@@ -276,6 +276,7 @@ Optional Dependencies
 * `Jinja2 <http://jinja.pocoo.org/>`__: Template engine for conditional HTML formatting.
 * `s3fs <http://s3fs.readthedocs.io/>`__: necessary for Amazon S3 access (s3fs >= 0.0.7).
 * `blosc <https://pypi.org/project/blosc>`__: for msgpack compression using ``blosc``
+* `gcsfs <http://gcsfs.readthedocs.io/>`__: necessary for Google Cloud Storage access (gcsfs >= 0.1.0).
 * One of
   `qtpy  <https://github.com/spyder-ide/qtpy>`__ (requires PyQt or PySide),
   `PyQt5 <https://www.riverbankcomputing.com/software/pyqt/download5>`__,

doc/source/whatsnew/v0.24.0.txt

@@ -18,7 +18,7 @@ Other Enhancements
 - :func:`Series.mode` and :func:`DataFrame.mode` now support the ``dropna`` parameter which can be used to specify whether NaN/NaT values should be considered (:issue:`17534`)
 - :func:`to_csv` now supports ``compression`` keyword when a file handle is passed. (:issue:`21227`)
 - :meth:`Index.droplevel` is now implemented also for flat indexes, for compatibility with MultiIndex (:issue:`21115`)
-
+- Added support for reading from Google Cloud Storage via the ``gcsfs`` library (:issue:`19454`)
 
 .. _whatsnew_0240.api_breaking:
 

pandas/io/common.py

@@ -88,7 +88,7 @@ def _is_url(url):
     """
     try:
         return parse_url(url).scheme in _VALID_URLS
-    except:
+    except Exception:
         return False
 
 
@@ -165,7 +165,15 @@ def is_s3_url(url):
     """Check for an s3, s3n, or s3a url"""
     try:
         return parse_url(url).scheme in ['s3', 's3n', 's3a']
-    except:  # noqa
+    except Exception:
+        return False
+
+
+def is_gcs_url(url):
+    """Check for a gcs url"""
+    try:
+        return parse_url(url).scheme in ['gcs', 'gs']
+    except Exception:
         return False
 
 
@@ -208,6 +216,13 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
                                          compression=compression,
                                          mode=mode)
 
+    if is_gcs_url(filepath_or_buffer):
+        from pandas.io import gcs
+        return gcs.get_filepath_or_buffer(filepath_or_buffer,
+                                          encoding=encoding,
+                                          compression=compression,
+                                          mode=mode)
+
     if isinstance(filepath_or_buffer, (compat.string_types,
                                        compat.binary_type,
                                        mmap.mmap)):

pandas/io/excel.py

@@ -46,7 +46,7 @@
 io : string, path object (pathlib.Path or py._path.local.LocalPath),
     file-like object, pandas ExcelFile, or xlrd workbook.
     The string could be a URL. Valid URL schemes include http, ftp, s3,
-    and file. For file URLs, a host is expected. For instance, a local
+    gcs, and file. For file URLs, a host is expected. For instance, a local
     file could be file://localhost/path/to/workbook.xlsx
 sheet_name : string, int, mixed list of strings/ints, or None, default 0
 

pandas/io/gcs.py

@@ -0,0 +1,16 @@
+""" GCS support for remote file interactivity """
+try:
+    import gcsfs
+except ImportError:
+    raise ImportError("The gcsfs library is required to handle GCS files")
+
+
+def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
+                           compression=None, mode=None):
+
+    if mode is None:
+        mode = 'rb'
+
+    fs = gcsfs.GCSFileSystem()
+    filepath_or_buffer = fs.open(filepath_or_buffer, mode)
+    return filepath_or_buffer, None, compression, True

pandas/io/json/json.py

@@ -231,9 +231,9 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
     Parameters
     ----------
     path_or_buf : a valid JSON string or file-like, default: None
-        The string could be a URL. Valid URL schemes include http, ftp, s3, and
-        file. For file URLs, a host is expected. For instance, a local file
-        could be ``file://localhost/path/to/table.json``
+        The string could be a URL. Valid URL schemes include http, ftp, s3,
+        gcs, and file. For file URLs, a host is expected. For instance, a local
+        file could be ``file://localhost/path/to/table.json``
 
     orient : string,
         Indication of expected JSON string format.

pandas/tests/io/test_gcs.py

@@ -0,0 +1,47 @@
+import numpy as np
+import pytest
+
+from pandas import DataFrame, date_range, read_csv
+from pandas.compat import StringIO
+from pandas.io.common import is_gcs_url
+from pandas.util import _test_decorators as td
+from pandas.util.testing import assert_frame_equal, patch
+
+
+def test_is_gcs_url():
+    assert is_gcs_url("gcs://pandas/somethingelse.com")
+    assert is_gcs_url("gs://pandas/somethingelse.com")
+    assert not is_gcs_url("s3://pandas/somethingelse.com")
+
+
+@td.skip_if_no('gcsfs')
+def test_read_csv_gcs():
+    df1 = DataFrame({'int': [1, 3], 'float': [2.0, np.nan], 'str': ['t', 's'],
+                     'dt': date_range('2018-06-18', periods=2)})
+    with patch('gcsfs.GCSFileSystem') as MockFileSystem:
+        instance = MockFileSystem.return_value
+        instance.open.return_value = StringIO(df1.to_csv(index=False))
+        df2 = read_csv('gs://test/test.csv', parse_dates=['dt'])
+
+    assert_frame_equal(df1, df2)
+
+
+@td.skip_if_no('gcsfs')
+def test_gcs_get_filepath_or_buffer():
+    df1 = DataFrame({'int': [1, 3], 'float': [2.0, np.nan], 'str': ['t', 's'],
+                     'dt': date_range('2018-06-18', periods=2)})
+    with patch('pandas.io.gcs.get_filepath_or_buffer') as MockGetFilepath:
+        MockGetFilepath.return_value = (StringIO(df1.to_csv(index=False)),
+                                        None, None, False)
+        df2 = read_csv('gs://test/test.csv', parse_dates=['dt'])
+
+    assert_frame_equal(df1, df2)
+    assert MockGetFilepath.called
+
+
+@pytest.mark.skipif(td.safe_import('gcsfs'),
+                    reason='Only check when gcsfs not installed')
+def test_gcs_not_present_exception():
+    with pytest.raises(ImportError) as e:
+        read_csv('gs://test/test.csv')
+        assert 'gcsfs library is required' in str(e.value)

pandas/tests/io/test_packers.py

@@ -682,7 +682,7 @@ def decompress(ob):
             np.dtype('timedelta64[ns]'): np.timedelta64(1, 'ns'),
         }
 
-        with patch(compress_module, 'decompress', decompress), \
+        with patch(compress + '.decompress', new=decompress), \
                 tm.assert_produces_warning(PerformanceWarning) as ws:
 
             i_rec = self.encode_decode(self.frame, compress=compress)

pandas/util/_print_versions.py

@@ -96,6 +96,7 @@ def show_versions(as_json=False):
         ("fastparquet", lambda mod: mod.__version__),
         ("pandas_gbq", lambda mod: mod.__version__),
         ("pandas_datareader", lambda mod: mod.__version__),
+        ("gcsfs", lambda mod: mod.__version__),
     ]
 
     deps_blob = list()

pandas/util/testing.py

@@ -48,6 +48,11 @@
 from pandas._libs import testing as _testing
 from pandas.io.common import urlopen
 
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
+
 
 N = 30
 K = 4
@@ -2667,58 +2672,6 @@ def _constructor(self):
         return SubclassedCategorical
 
 
-@contextmanager
-def patch(ob, attr, value):
-    """Temporarily patch an attribute of an object.
-
-    Parameters
-    ----------
-    ob : any
-        The object to patch. This must support attribute assignment for `attr`.
-    attr : str
-        The name of the attribute to patch.
-    value : any
-        The temporary attribute to assign.
-
-    Examples
-    --------
-    >>> class C(object):
-    ...     attribute = 'original'
-    ...
-    >>> C.attribute
-    'original'
-    >>> with patch(C, 'attribute', 'patched'):
-    ...     in_context = C.attribute
-    ...
-    >>> in_context
-    'patched'
-    >>> C.attribute  # the value is reset when the context manager exists
-    'original'
-
-    Correctly replaces attribute when the manager exits with an exception.
-    >>> with patch(C, 'attribute', 'patched'):
-    ...     in_context = C.attribute
-    ...     raise ValueError()
-    Traceback (most recent call last):
-       ...
-    ValueError
-    >>> in_context
-    'patched'
-    >>> C.attribute
-    'original'
-    """
-    noattr = object()  # mark that the attribute never existed
-    old = getattr(ob, attr, noattr)
-    setattr(ob, attr, value)
-    try:
-        yield
-    finally:
-        if old is noattr:
-            delattr(ob, attr)
-        else:
-            setattr(ob, attr, old)
-
-
 @contextmanager
 def set_timezone(tz):
     """Context manager for temporarily setting a timezone.