Merge branch 'master' into readjson-lines-chunks

Author: fangchenli

.devcontainer.json

@@ -17,7 +17,9 @@
 		"python.linting.pylintEnabled": false,
 		"python.linting.mypyEnabled": true,
 		"python.testing.pytestEnabled": true,
-		"python.testing.cwd": "pandas/tests"
+		"python.testing.pytestArgs": [
+			"pandas"
+		]
 	},
 
 	// Add the IDs of extensions you want installed when the container is created in the array below.

asv_bench/benchmarks/groupby.py

@@ -16,7 +16,7 @@
 
 from .pandas_vb_common import tm
 
-method_blacklist = {
+method_blocklist = {
     "object": {
         "median",
         "prod",
@@ -403,7 +403,7 @@ class GroupByMethods:
     ]
 
     def setup(self, dtype, method, application):
-        if method in method_blacklist.get(dtype, {}):
+        if method in method_blocklist.get(dtype, {}):
             raise NotImplementedError  # skip benchmark
         ngroups = 1000
         size = ngroups * 2

ci/code_checks.sh

@@ -248,19 +248,19 @@ fi
 ### CODE ###
 if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then
 
-    MSG='Check import. No warnings, and blacklist some optional dependencies' ; echo $MSG
+    MSG='Check import. No warnings, and blocklist some optional dependencies' ; echo $MSG
     python -W error -c "
 import sys
 import pandas
 
-blacklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
+blocklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
              'lxml', 'matplotlib', 'numexpr', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
              'tables', 'urllib.request', 'xlrd', 'xlsxwriter', 'xlwt'}
 
 # GH#28227 for some of these check for top-level modules, while others are
 #  more specific (e.g. urllib.request)
 import_mods = set(m.split('.')[0] for m in sys.modules) | set(sys.modules)
-mods = blacklist & import_mods
+mods = blocklist & import_mods
 if mods:
     sys.stderr.write('err: pandas should not import: {}\n'.format(', '.join(mods)))
     sys.exit(len(mods))

ci/deps/azure-36-locale.yaml

@@ -15,7 +15,6 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - gcsfs
   - html5lib
   - ipython
   - jinja2
@@ -31,7 +30,6 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
   - scipy
   - xarray
   - xlrd

ci/deps/azure-37-locale.yaml

@@ -27,7 +27,6 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
   - scipy
   - xarray
   - xlrd

ci/deps/azure-windows-37.yaml

@@ -15,7 +15,8 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
-  - gcsfs
+  - fsspec>=0.7.4
+  - gcsfs>=0.6.0
   - html5lib
   - jinja2
   - lxml
@@ -28,7 +29,7 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - scipy
   - sqlalchemy
   - xlrd

ci/deps/travis-36-cov.yaml

@@ -18,7 +18,8 @@ dependencies:
   - cython>=0.29.16
   - dask
   - fastparquet>=0.3.2
-  - gcsfs
+  - fsspec>=0.7.4
+  - gcsfs>=0.6.0
   - geopandas
   - html5lib
   - matplotlib
@@ -35,7 +36,7 @@ dependencies:
   - pytables
   - python-snappy
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - scikit-learn
   - scipy
   - sqlalchemy

ci/deps/travis-36-locale.yaml

@@ -16,7 +16,6 @@ dependencies:
   - blosc=1.14.3
   - python-blosc
   - fastparquet=0.3.2
-  - gcsfs=0.2.2
   - html5lib
   - ipython
   - jinja2
@@ -33,7 +32,6 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs=0.3.0
   - scipy
   - sqlalchemy=1.1.4
   - xarray=0.10

ci/deps/travis-36-slow.yaml

@@ -13,6 +13,7 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
+  - fsspec>=0.7.4
   - html5lib
   - lxml
   - matplotlib
@@ -25,7 +26,7 @@ dependencies:
   - pytables
   - python-dateutil
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - scipy
   - sqlalchemy
   - xlrd

ci/deps/travis-37.yaml

@@ -13,12 +13,13 @@ dependencies:
 
   # pandas dependencies
   - botocore>=1.11
+  - fsspec>=0.7.4
   - numpy
   - python-dateutil
   - nomkl
   - pyarrow
   - pytz
-  - s3fs
+  - s3fs>=0.4.0
   - tabulate
   - pyreadstat
   - pip

doc/source/getting_started/install.rst

@@ -267,8 +267,9 @@ SQLAlchemy                1.1.4              SQL support for databases other tha
 SciPy                     0.19.0             Miscellaneous statistical functions
 XLsxWriter                0.9.8              Excel writing
 blosc                                        Compression for HDF5
+fsspec                    0.7.4              Handling files aside from local and HTTP
 fastparquet               0.3.2              Parquet reading / writing
-gcsfs                     0.2.2              Google Cloud Storage access
+gcsfs                     0.6.0              Google Cloud Storage access
 html5lib                                     HTML parser for read_html (see :ref:`note <optional_html>`)
 lxml                      3.8.0              HTML parser for read_html (see :ref:`note <optional_html>`)
 matplotlib                2.2.2              Visualization
@@ -282,7 +283,7 @@ pyreadstat                                   SPSS files (.sav) reading
 pytables                  3.4.3              HDF5 reading / writing
 pyxlsb                    1.0.6              Reading for xlsb files
 qtpy                                         Clipboard I/O
-s3fs                      0.3.0              Amazon S3 access
+s3fs                      0.4.0              Amazon S3 access
 tabulate                  0.8.3              Printing in Markdown-friendly format (see `tabulate`_)
 xarray                    0.8.2              pandas-like API for N-dimensional data
 xclip                                        Clipboard I/O on linux

doc/source/user_guide/advanced.rst

@@ -260,7 +260,9 @@ You don't have to specify all levels of the ``MultiIndex`` by passing only the
 first elements of the tuple. For example, you can use "partial" indexing to
 get all elements with ``bar`` in the first level as follows:
 
-df.loc['bar']
+.. ipython:: python
+
+   df.loc['bar']
 
 This is a shortcut for the slightly more verbose notation ``df.loc[('bar',),]`` (equivalent
 to ``df.loc['bar',]`` in this example).

doc/source/whatsnew/v0.14.1.rst

@@ -131,7 +131,7 @@ Enhancements
 
 - Implemented ``sem`` (standard error of the mean) operation for ``Series``,
   ``DataFrame``, ``Panel``, and ``Groupby`` (:issue:`6897`)
-- Add ``nlargest`` and ``nsmallest`` to the ``Series`` ``groupby`` whitelist,
+- Add ``nlargest`` and ``nsmallest`` to the ``Series`` ``groupby`` allowlist,
   which means you can now use these methods on a ``SeriesGroupBy`` object
   (:issue:`7053`).
 - All offsets ``apply``, ``rollforward`` and ``rollback`` can now handle ``np.datetime64``, previously results in ``ApplyTypeError`` (:issue:`7452`)

doc/source/whatsnew/v1.1.0.rst

@@ -245,6 +245,22 @@ If needed you can adjust the bins with the argument ``offset`` (a Timedelta) tha
 
 For a full example, see: :ref:`timeseries.adjust-the-start-of-the-bins`.
 
+fsspec now used for filesystem handling
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+For reading and writing to filesystems other than local and reading from HTTP(S),
+the optional dependency ``fsspec`` will be used to dispatch operations (:issue:`33452`).
+This will give unchanged
+functionality for S3 and GCS storage, which were already supported, but also add
+support for several other storage implementations such as `Azure Datalake and Blob`_,
+SSH, FTP, dropbox and github. For docs and capabilities, see the `fsspec docs`_.
+
+The existing capability to interface with S3 and GCS will be unaffected by this
+change, as ``fsspec`` will still bring in the same packages as before.
+
+.. _Azure Datalake and Blob: https://github.com/dask/adlfs
+
+.. _fsspec docs: https://filesystem-spec.readthedocs.io/en/latest/
 
 .. _whatsnew_110.enhancements.other:
 
@@ -292,12 +308,15 @@ Other enhancements
 - :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` now accept an ``errors`` argument (:issue:`22610`)
 - :meth:`groupby.transform` now allows ``func`` to be ``pad``, ``backfill`` and ``cumcount`` (:issue:`31269`).
 - :meth:`~pandas.io.json.read_json` now accepts `nrows` parameter. (:issue:`33916`).
+- :meth:`DataFrame.hist`, :meth:`Series.hist`, :meth:`core.groupby.DataFrameGroupBy.hist`, and :meth:`core.groupby.SeriesGroupBy.hist` have gained the ``legend`` argument. Set to True to show a legend in the histogram. (:issue:`6279`)
 - :func:`concat` and :meth:`~DataFrame.append` now preserve extension dtypes, for example
   combining a nullable integer column with a numpy integer column will no longer
   result in object dtype but preserve the integer dtype (:issue:`33607`, :issue:`34339`).
 - :meth:`~pandas.io.gbq.read_gbq` now allows to disable progress bar (:issue:`33360`).
 - :meth:`~pandas.io.gbq.read_gbq` now supports the ``max_results`` kwarg from ``pandas-gbq`` (:issue:`34639`).
+- :meth:`Dataframe.cov` and :meth:`Series.cov` now support a new parameter ddof to support delta degrees of freedom as in the corresponding numpy methods (:issue:`34611`).
 - :meth:`DataFrame.to_html` and :meth:`DataFrame.to_string`'s ``col_space`` parameter now accepts a list of dict to change only some specific columns' width (:issue:`28917`).
+- :meth:`DataFrame.to_excel` can now also write OpenOffice spreadsheet (.ods) files (:issue:`27222`)
 
 .. ---------------------------------------------------------------------------
 
@@ -700,7 +719,9 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | fastparquet     | 0.3.2           |         |
 +-----------------+-----------------+---------+
-| gcsfs           | 0.2.2           |         |
+| fsspec          | 0.7.4           |         |
++-----------------+-----------------+---------+
+| gcsfs           | 0.6.0           |    X    |
 +-----------------+-----------------+---------+
 | lxml            | 3.8.0           |         |
 +-----------------+-----------------+---------+
@@ -716,7 +737,7 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | pytables        | 3.4.3           |    X    |
 +-----------------+-----------------+---------+
-| s3fs            | 0.3.0           |         |
+| s3fs            | 0.4.0           |    X    |
 +-----------------+-----------------+---------+
 | scipy           | 1.2.0           |    X    |
 +-----------------+-----------------+---------+
@@ -933,6 +954,7 @@ Indexing
 - Bug in :meth:`Series.at` when used with a :class:`MultiIndex` would raise an exception on valid inputs (:issue:`26989`)
 - Bug in :meth:`DataFrame.loc` with dictionary of values changes columns with dtype of ``int`` to ``float`` (:issue:`34573`)
 - Bug in :meth:`Series.loc` when used with a :class:`MultiIndex` would raise an IndexingError when accessing a None value (:issue:`34318`)
+- Bug in :meth:`DataFrame.reset_index` and :meth:`Series.reset_index` would not preserve data types on an empty :class:`DataFrame` or :class:`Series` with a :class:`MultiIndex` (:issue:`19602`)
 
 Missing
 ^^^^^^^
@@ -997,7 +1019,7 @@ I/O
 - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`)
 - Bug in :meth:`~pandas.io.stata.StataReader` which resulted in categorical variables with difference dtypes when reading data using an iterator. (:issue:`31544`)
 - :meth:`HDFStore.keys` has now an optional `include` parameter that allows the retrieval of all native HDF5 table names (:issue:`29916`)
-- :meth:`read_json` now could read line-delimited JSON from file url. (:issue:`27135`)
+- Bug in :meth:`read_excel` for ODS files removes 0.0 values (:issue:`27222`)
 
 Plotting
 ^^^^^^^^
@@ -1008,6 +1030,7 @@ Plotting
 - Bug in :meth:`DataFrame.hist` where the order of ``column`` argument was ignored (:issue:`29235`)
 - Bug in :meth:`DataFrame.plot.scatter` that when adding multiple plots with different ``cmap``, colorbars alway use the first ``cmap`` (:issue:`33389`)
 - Bug in :meth:`DataFrame.plot.scatter` was adding a colorbar to the plot even if the argument `c` was assigned to a column containing color names (:issue:`34316`)
+- Bug in :meth:`pandas.plotting.bootstrap_plot` was causing cluttered axes and overlapping labels (:issue:`34905`)
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1100,6 +1123,7 @@ Other
 - :class:`IntegerArray` now implements the ``sum`` operation (:issue:`33172`)
 - Bug in :class:`Tick` comparisons raising ``TypeError`` when comparing against timedelta-like objects (:issue:`34088`)
 - Bug in :class:`Tick` multiplication raising ``TypeError`` when multiplying by a float (:issue:`34486`)
+- Passing a `set` as `names` argument to :func:`pandas.read_csv`, :func:`pandas.read_table`, or :func:`pandas.read_fwf` will raise ``ValueError: Names should be an ordered collection.`` (:issue:`34946`)
 
 .. ---------------------------------------------------------------------------
 

environment.yml

@@ -98,7 +98,9 @@ dependencies:
 
   - pyqt>=5.9.2  # pandas.read_clipboard
   - pytables>=3.4.3  # pandas.read_hdf, DataFrame.to_hdf
-  - s3fs  # pandas.read_csv... when using 's3://...' path
+  - s3fs>=0.4.0  # file IO when using 's3://...' path
+  - fsspec>=0.7.4  # for generic remote file operations
+  - gcsfs>=0.6.0  # file IO when using 'gcs://...' path
   - sqlalchemy  # pandas.read_sql, DataFrame.to_sql
   - xarray  # DataFrame.to_xarray
   - cftime  # Needed for downstream xarray.CFTimeIndex test

pandas/_libs/window/aggregations.pyx

@@ -1377,17 +1377,11 @@ def roll_generic_fixed(object obj,
                 output[i] = NaN
 
         # remaining full-length windows
-        buf = <float64_t *>arr.data
-        bufarr = np.empty(win, dtype=float)
-        oldbuf = <float64_t *>bufarr.data
-        for i in range((win - offset), (N - offset)):
-            buf = buf + 1
-            bufarr.data = <char *>buf
+        for j, i in enumerate(range((win - offset), (N - offset)), 1):
             if counts[i] >= minp:
-                output[i] = func(bufarr, *args, **kwargs)
+                output[i] = func(arr[j:j + win], *args, **kwargs)
             else:
                 output[i] = NaN
-        bufarr.data = <char *>oldbuf
 
         # truncated windows at the end
         for i in range(int_max(N - offset, 0), N):

pandas/compat/_optional.py

@@ -8,8 +8,9 @@
 VERSIONS = {
     "bs4": "4.6.0",
     "bottleneck": "1.2.1",
+    "fsspec": "0.7.4",
     "fastparquet": "0.3.2",
-    "gcsfs": "0.2.2",
+    "gcsfs": "0.6.0",
     "lxml.etree": "3.8.0",
     "matplotlib": "2.2.2",
     "numexpr": "2.6.2",
@@ -20,7 +21,7 @@
     "pytables": "3.4.3",
     "pytest": "5.0.1",
     "pyxlsb": "1.0.6",
-    "s3fs": "0.3.0",
+    "s3fs": "0.4.0",
     "scipy": "1.2.0",
     "sqlalchemy": "1.1.4",
     "tables": "3.4.3",

pandas/core/common.py

@@ -404,7 +404,7 @@ def random_state(state=None):
         If receives `None`, returns np.random.
         If receives anything else, raises an informative ValueError.
 
-        ..versionchanged:: 1.1.0
+        .. versionchanged:: 1.1.0
 
             array-like and BitGenerator (for NumPy>=1.18) object now passed to
             np.random.RandomState() as seed

pandas/core/config_init.py

@@ -553,6 +553,7 @@ def use_inf_as_na_cb(key):
 _xls_options = ["xlwt"]
 _xlsm_options = ["openpyxl"]
 _xlsx_options = ["openpyxl", "xlsxwriter"]
+_ods_options = ["odf"]
 
 
 with cf.config_prefix("io.excel.xls"):
@@ -581,6 +582,15 @@ def use_inf_as_na_cb(key):
     )
 
 
+with cf.config_prefix("io.excel.ods"):
+    cf.register_option(
+        "writer",
+        "auto",
+        writer_engine_doc.format(ext="ods", others=", ".join(_ods_options)),
+        validator=str,
+    )
+
+
 # Set up the io.parquet specific configuration.
 parquet_engine_doc = """
 : string