Merge remote-tracking branch 'upstream/master' into Interval-array

Author: sparalic

README.md

@@ -233,3 +233,5 @@ You can also triage issues which may include reproducing bug reports, or asking
 Or maybe through using pandas you have an idea of your own or are looking for something in the documentation and thinking ‘this can be improved’...you can do something about it!
 
 Feel free to ask questions on the [mailing list](https://groups.google.com/forum/?fromgroups#!forum/pydata) or on [Gitter](https://gitter.im/pydata/pandas).
+
+As contributors and maintainers to this project, you are expected to abide by pandas' code of conduct. More information can be found at: [Contributor Code of Conduct](https://github.com/pandas-dev/pandas/blob/master/.github/CODE_OF_CONDUCT.md)

ci/deps/azure-37-locale.yaml

@@ -26,8 +26,8 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.1
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pytest-azurepipelines
   - pip

ci/deps/azure-37-numpydev.yaml

@@ -6,7 +6,8 @@ dependencies:
   - pytz
   - Cython>=0.28.2
   # universal
-  - pytest>=4.0.2
+  # pytest < 5 until defaults has pytest-xdist>=1.29.0
+  - pytest>=4.0.2,<5.0
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0

ci/deps/azure-macos-35.yaml

@@ -25,8 +25,8 @@ dependencies:
   - pip:
     - pyreadstat
     # universal
-    - pytest==4.5.0
-    - pytest-xdist
+    - pytest>=5.0.1
+    - pytest-xdist>=1.29.0
     - pytest-mock
     - hypothesis>=3.58.0
     # https://github.com/pandas-dev/pandas/issues/27421

ci/deps/azure-windows-36.yaml

@@ -23,8 +23,8 @@ dependencies:
   - xlwt
   # universal
   - cython>=0.28.2
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.1
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pytest-azurepipelines
   - hypothesis>=3.58.0

ci/deps/azure-windows-37.yaml

@@ -26,8 +26,8 @@ dependencies:
   - xlwt
   # universal
   - cython>=0.28.2
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.0
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - pytest-azurepipelines
   - hypothesis>=3.58.0

ci/deps/travis-36-cov.yaml

@@ -39,8 +39,8 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
-  - pytest-xdist
+  - pytest>=5.0.1
+  - pytest-xdist>=1.29.0
   - pytest-cov
   - pytest-mock
   - hypothesis>=3.58.0

ci/deps/travis-37.yaml

@@ -13,8 +13,8 @@ dependencies:
   - pyarrow
   - pytz
   # universal
-  - pytest>=4.0.2
-  - pytest-xdist
+  - pytest>=5.0.0
+  - pytest-xdist>=1.29.0
   - pytest-mock
   - hypothesis>=3.58.0
   - s3fs

doc/source/user_guide/io.rst

@@ -28,6 +28,7 @@ The pandas I/O API is a set of top level ``reader`` functions accessed like
     :delim: ;
 
     text;`CSV <https://en.wikipedia.org/wiki/Comma-separated_values>`__;:ref:`read_csv<io.read_csv_table>`;:ref:`to_csv<io.store_in_csv>`
+    text;Fixed-Width Text File;:ref:`read_fwf<io.fwf_reader>`
     text;`JSON <https://www.json.org/>`__;:ref:`read_json<io.json_reader>`;:ref:`to_json<io.json_writer>`
     text;`HTML <https://en.wikipedia.org/wiki/HTML>`__;:ref:`read_html<io.read_html>`;:ref:`to_html<io.html>`
     text; Local clipboard;:ref:`read_clipboard<io.clipboard>`;:ref:`to_clipboard<io.clipboard>`
@@ -1372,6 +1373,7 @@ should pass the ``escapechar`` option:
    print(data)
    pd.read_csv(StringIO(data), escapechar='\\')
 
+.. _io.fwf_reader:
 .. _io.fwf:
 
 Files with fixed width columns
@@ -3572,7 +3574,7 @@ Closing a Store and using a context manager:
 Read/write API
 ''''''''''''''
 
-``HDFStore`` supports an top-level API using  ``read_hdf`` for reading and ``to_hdf`` for writing,
+``HDFStore`` supports a top-level API using  ``read_hdf`` for reading and ``to_hdf`` for writing,
 similar to how ``read_csv`` and ``to_csv`` work.
 
 .. ipython:: python
@@ -3687,7 +3689,7 @@ Hierarchical keys
 Keys to a store can be specified as a string. These can be in a
 hierarchical path-name like format (e.g. ``foo/bar/bah``), which will
 generate a hierarchy of sub-stores (or ``Groups`` in PyTables
-parlance). Keys can be specified with out the leading '/' and are **always**
+parlance). Keys can be specified without the leading '/' and are **always**
 absolute (e.g. 'foo' refers to '/foo'). Removal operations can remove
 everything in the sub-store and **below**, so be *careful*.
 
@@ -3825,7 +3827,7 @@ data.
 
 A query is specified using the ``Term`` class under the hood, as a boolean expression.
 
-* ``index`` and ``columns`` are supported indexers of a ``DataFrames``.
+* ``index`` and ``columns`` are supported indexers of ``DataFrames``.
 * if ``data_columns`` are specified, these can be used as additional indexers.
 
 Valid comparison operators are:
@@ -3917,7 +3919,7 @@ Use boolean expressions, with in-line function evaluation.
 
     store.select('dfq', "index>pd.Timestamp('20130104') & columns=['A', 'B']")
 
-Use and inline column reference
+Use inline column reference.
 
 .. ipython:: python
 
@@ -4593,8 +4595,8 @@ Performance
   write chunksize (default is 50000). This will significantly lower
   your memory usage on writing.
 * You can pass ``expectedrows=<int>`` to the first ``append``,
-  to set the TOTAL number of expected rows that ``PyTables`` will
-  expected. This will optimize read/write performance.
+  to set the TOTAL number of rows that ``PyTables`` will expect.
+  This will optimize read/write performance.
 * Duplicate rows can be written to tables, but are filtered out in
   selection (with the last items being selected; thus a table is
   unique on major, minor pairs)

doc/source/whatsnew/v0.25.1.rst

@@ -25,14 +25,13 @@ Bug fixes
 Categorical
 ^^^^^^^^^^^
 
--
--
+- Bug in :meth:`Categorical.fillna` would replace all values, not just those that are ``NaN`` (:issue:`26215`)
 -
 
 Datetimelike
 ^^^^^^^^^^^^
 - Bug in :func:`to_datetime` where passing a timezone-naive :class:`DatetimeArray` or :class:`DatetimeIndex` and ``utc=True`` would incorrectly return a timezone-naive result (:issue:`27733`)
--
+- Bug in :meth:`Period.to_timestamp` where a :class:`Period` outside the :class:`Timestamp` implementation bounds (roughly 1677-09-21 to 2262-04-11) would return an incorrect :class:`Timestamp` instead of raising ``OutOfBoundsDatetime`` (:issue:`19643`)
 -
 -
 
@@ -54,8 +53,8 @@ Numeric
 ^^^^^^^
 - Bug in :meth:`Series.interpolate` when using a timezone aware :class:`DatetimeIndex` (:issue:`27548`)
 - Bug when printing negative floating point complex numbers would raise an ``IndexError`` (:issue:`27484`)
--
--
+- Bug where :class:`DataFrame` arithmetic operators such as :meth:`DataFrame.mul` with a :class:`Series` with axis=1 would raise an ``AttributeError`` on :class:`DataFrame` larger than the minimum threshold to invoke numexpr (:issue:`27636`)
+- Bug in :class:`DataFrame` arithmetic where missing values in results were incorrectly masked with ``NaN`` instead of ``Inf`` (:issue:`27464`)
 
 Conversion
 ^^^^^^^^^^
@@ -83,14 +82,15 @@ Indexing
 ^^^^^^^^
 
 - Bug in partial-string indexing returning a NumPy array rather than a ``Series`` when indexing with a scalar like ``.loc['2015']`` (:issue:`27516`)
-- Break reference cycle involving :class:`Index` to allow garbage collection of :class:`Index` objects without running the GC. (:issue:`27585`)
--
+- Break reference cycle involving :class:`Index` and other index classes to allow garbage collection of index objects without running the GC. (:issue:`27585`, :issue:`27840`)
+- Fix regression in assigning values to a single column of a DataFrame with a ``MultiIndex`` columns (:issue:`27841`).
+- Fix regression in ``.ix`` fallback with an ``IntervalIndex`` (:issue:`27865`).
 -
 
 Missing
 ^^^^^^^
 
--
+- Bug in :func:`pandas.isnull` or :func:`pandas.isna` when the input is a type e.g. `type(pandas.Series())` (:issue:`27482`)
 -
 -
 
@@ -103,37 +103,41 @@ MultiIndex
 
 I/O
 ^^^
-
--
--
+- Avoid calling ``S3File.s3`` when reading parquet, as this was removed in s3fs version 0.3.0 (:issue:`27756`)
+- Better error message when a negative header is passed in :func:`pandas.read_csv` (:issue:`27779`)
+- Follow the ``min_rows`` display option (introduced in v0.25.0) correctly in the html repr in the notebook (:issue:`27991`).
 -
 
 Plotting
 ^^^^^^^^
 
 - Added a pandas_plotting_backends entrypoint group for registering plot backends. See :ref:`extending.plotting-backends` for more (:issue:`26747`).
+- Fixed the re-instatement of Matplotlib datetime converters after calling
+  `pandas.plotting.deregister_matplotlib_converters()` (:issue:`27481`).
+-
 - Fix compatibility issue with matplotlib when passing a pandas ``Index`` to a plot call (:issue:`27775`).
 -
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
 - Bug in :meth:`pandas.core.groupby.DataFrameGroupBy.transform` where applying a timezone conversion lambda function would drop timezone information (:issue:`27496`)
+- Bug in :meth:`pandas.core.groupby.GroupBy.nth` where ``observed=False`` was being ignored for Categorical groupers (:issue:`26385`)
 - Bug in windowing over read-only arrays (:issue:`27766`)
--
+- Fixed segfault in `pandas.core.groupby.DataFrameGroupBy.quantile` when an invalid quantile was passed (:issue:`27470`)
 -
 
 Reshaping
 ^^^^^^^^^
 
 - A ``KeyError`` is now raised if ``.unstack()`` is called on a :class:`Series` or :class:`DataFrame` with a flat :class:`Index` passing a name which is not the correct one (:issue:`18303`)
--  Bug in :meth:`DataFrame.crosstab` when ``margins`` set to ``True`` and ``normalize`` is not ``False``, an error is raised. (:issue:`27500`)
+- Bug in :meth:`DataFrame.crosstab` when ``margins`` set to ``True`` and ``normalize`` is not ``False``, an error is raised. (:issue:`27500`)
 - :meth:`DataFrame.join` now suppresses the ``FutureWarning`` when the sort parameter is specified (:issue:`21952`)
--
+- Bug in :meth:`DataFrame.join` raising with readonly arrays (:issue:`27943`)
 
 Sparse
 ^^^^^^
-
+- Bug in reductions for :class:`Series` with Sparse dtypes (:issue:`27080`)
 -
 -
 -
@@ -160,6 +164,14 @@ Other
 -
 -
 
+I/O and LZMA
+~~~~~~~~~~~~
+
+Some users may unknowingly have an incomplete Python installation, which lacks the `lzma` module from the standard library. In this case, `import pandas` failed due to an `ImportError` (:issue: `27575`).
+Pandas will now warn, rather than raising an `ImportError` if the `lzma` module is not present. Any subsequent attempt to use `lzma` methods will raise a `RuntimeError`.
+A possible fix for the lack of the `lzma` module is to ensure you have the necessary libraries and then re-install Python.
+For example, on MacOS installing Python with `pyenv` may lead to an incomplete Python installation due to unmet system dependencies at compilation time (like `xz`). Compilation will succeed, but Python might fail at run time. The issue can be solved by installing the necessary dependencies and then re-installing Python.
+
 .. _whatsnew_0.251.contributors:
 
 Contributors

doc/source/whatsnew/v0.7.3.rst

@@ -25,8 +25,6 @@ New features
    from pandas.tools.plotting import scatter_matrix
    scatter_matrix(df, alpha=0.2)        # noqa F821
 
-.. image:: ../savefig/scatter_matrix_kde.png
-   :width: 5in
 
 - Add ``stacked`` argument to Series and DataFrame's ``plot`` method for
   :ref:`stacked bar plots <visualization.barplot>`.
@@ -35,15 +33,11 @@ New features
 
    df.plot(kind='bar', stacked=True)    # noqa F821
 
-.. image:: ../savefig/bar_plot_stacked_ex.png
-   :width: 4in
 
 .. code-block:: python
 
    df.plot(kind='barh', stacked=True)   # noqa F821
 
-.. image:: ../savefig/barh_plot_stacked_ex.png
-   :width: 4in
 
 - Add log x and y :ref:`scaling options <visualization.basic>` to
   ``DataFrame.plot`` and ``Series.plot``

doc/source/whatsnew/v1.0.0.rst

@@ -21,27 +21,27 @@ including other versions of pandas.
 Enhancements
 ~~~~~~~~~~~~
 
-.. _whatsnew_1000.enhancements.other:
-
 -
 -
 
+.. _whatsnew_1000.enhancements.other:
+
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
 
-.. _whatsnew_1000.api_breaking:
-
 -
 -
 
+.. _whatsnew_1000.api_breaking:
+
 Backwards incompatible API changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. _whatsnew_1000.api.other:
-
 - :class:`pandas.core.groupby.GroupBy.transform` now raises on invalid operation names (:issue:`27489`).
 -
 
+.. _whatsnew_1000.api.other:
+
 Other API changes
 ^^^^^^^^^^^^^^^^^
 
@@ -87,6 +87,7 @@ Bug fixes
 Categorical
 ^^^^^^^^^^^
 
+- Added test to assert the :func:`fillna` raises the correct ValueError message when the value isn't a value from categories (:issue:`13628`)
 -
 -
 
@@ -157,14 +158,17 @@ MultiIndex
 I/O
 ^^^
 
--
+- :meth:`read_csv` now accepts binary mode file buffers when using the Python csv engine (:issue:`23779`)
 -
 
 Plotting
 ^^^^^^^^
 
+- Bug in :meth:`Series.plot` not able to plot boolean values (:issue:`23719`)
 -
--
+- Bug in :meth:`DataFrame.plot` producing incorrect legend markers when plotting multiple series on the same axis (:issue:`18222`)
+- Bug in :meth:`DataFrame.plot` when ``kind='box'`` and data contains datetime or timedelta data. These types are now automatically dropped (:issue:`22799`)
+- Bug in :meth:`DataFrame.plot.line` and :meth:`DataFrame.plot.area` produce wrong xlim in x-axis (:issue:`27686`, :issue:`25160`, :issue:`24784`)
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^

pandas/_libs/groupby.pyx

@@ -719,6 +719,11 @@ def group_quantile(ndarray[float64_t] out,
         ndarray[int64_t] counts, non_na_counts, sort_arr
 
     assert values.shape[0] == N
+
+    if not (0 <= q <= 1):
+        raise ValueError("'q' must be between 0 and 1. Got"
+                         " '{}' instead".format(q))
+
     inter_methods = {
         'linear': INTERPOLATION_LINEAR,
         'lower': INTERPOLATION_LOWER,

pandas/_libs/hashtable.pyx

@@ -108,7 +108,7 @@ cdef class Int64Factorizer:
     def get_count(self):
         return self.count
 
-    def factorize(self, int64_t[:] values, sort=False,
+    def factorize(self, const int64_t[:] values, sort=False,
                   na_sentinel=-1, na_value=None):
         """
         Factorize values with nans replaced by na_sentinel

pandas/_libs/parsers.pyx

@@ -2,7 +2,6 @@
 # See LICENSE for the license
 import bz2
 import gzip
-import lzma
 import os
 import sys
 import time
@@ -59,9 +58,12 @@ from pandas.core.arrays import Categorical
 from pandas.core.dtypes.concat import union_categoricals
 import pandas.io.common as icom
 
+from pandas.compat import _import_lzma, _get_lzma_file
 from pandas.errors import (ParserError, DtypeWarning,
                            EmptyDataError, ParserWarning)
 
+lzma = _import_lzma()
+
 # Import CParserError as alias of ParserError for backwards compatibility.
 # Ultimately, we want to remove this import. See gh-12665 and gh-14479.
 CParserError = ParserError
@@ -645,9 +647,9 @@ cdef class TextReader:
                                      'zip file %s', str(zip_names))
             elif self.compression == 'xz':
                 if isinstance(source, str):
-                    source = lzma.LZMAFile(source, 'rb')
+                    source = _get_lzma_file(lzma)(source, 'rb')
                 else:
-                    source = lzma.LZMAFile(filename=source)
+                    source = _get_lzma_file(lzma)(filename=source)
             else:
                 raise ValueError('Unrecognized compression type: %s' %
                                  self.compression)