Merge branch 'master' of https://github.com/pandas-dev/pandas into new_udfs_list_agg

� Conflicts:
�	pandas/core/groupby/generic.py
�	pandas/core/groupby/groupby.py
�	pandas/tests/apply/test_frame_apply.py
�	pandas/tests/groupby/test_groupby.py

Author: rhshadrach

.github/workflows/ci.yml

@@ -45,6 +45,15 @@ jobs:
         environment-file: ${{ env.ENV_FILE }}
         use-only-tar-bz2: true
 
+    - name: Install node.js (for pyright)
+      uses: actions/setup-node@v2
+      with:
+        node-version: "16"
+
+    - name: Install pyright
+      # note: keep version in sync with .pre-commit-config.yaml
+      run: npm install -g [email protected]
+
     - name: Build Pandas
       uses: ./.github/actions/build_pandas
 

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/python/black
-    rev: 21.7b0
+    rev: 21.9b0
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
@@ -58,7 +58,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.23.3
+    rev: v2.29.0
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -89,7 +89,9 @@ repos:
         language: node
         pass_filenames: false
         types: [python]
-        additional_dependencies: ['[email protected]']
+        stages: [manual]
+        # note: keep version in sync with .github/workflows/ci.yml
+        additional_dependencies: ['[email protected]']
 -   repo: local
     hooks:
     -   id: flake8-rst

asv_bench/benchmarks/indexing_engines.py

@@ -1,5 +1,5 @@
 """
-Benchmarks in this fiel depend exclusively on code in _libs/
+Benchmarks in this file depend exclusively on code in _libs/
 
 If a PR does not edit anything in _libs, it is very unlikely that benchmarks
 in this file will be affected.

asv_bench/benchmarks/sparse.py

@@ -95,11 +95,11 @@ class ToCooFrame:
     def setup(self):
         N = 10000
         k = 10
-        arr = np.full((N, k), np.nan)
+        arr = np.zeros((N, k), dtype=float)
         arr[0, 0] = 3.0
         arr[12, 7] = -1.0
         arr[0, 9] = 11.2
-        self.df = pd.DataFrame(arr, dtype=pd.SparseDtype("float"))
+        self.df = pd.DataFrame(arr, dtype=pd.SparseDtype("float", fill_value=0.0))
 
     def time_to_coo(self):
         self.df.sparse.to_coo()
@@ -195,4 +195,17 @@ def time_take(self, indices, allow_fill):
         self.sp_arr.take(indices, allow_fill=allow_fill)
 
 
+class GetItem:
+    def setup(self):
+        N = 1_000_000
+        arr = make_array(N, 1e-5, np.nan, np.float64)
+        self.sp_arr = SparseArray(arr)
+
+    def time_integer_indexing(self):
+        self.sp_arr[78]
+
+    def time_slice(self):
+        self.sp_arr[1:]
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

ci/code_checks.sh

@@ -84,6 +84,10 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
       pandas/tseries/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Cython Doctests' ; echo $MSG
+    python -m pytest --doctest-cython pandas/_libs
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
 fi
 
 ### DOCSTRINGS ###
@@ -104,6 +108,13 @@ if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then
     MSG='Performing static analysis using mypy' ; echo $MSG
     mypy pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    # run pyright, if it is installed
+    if command -v pyright &> /dev/null ; then
+        MSG='Performing static analysis using pyright' ; echo $MSG
+        pyright
+        RET=$(($RET + $?)) ; echo $MSG "DONE"
+    fi
 fi
 
 exit $RET

ci/deps/actions-38-db.yaml

@@ -16,7 +16,7 @@ dependencies:
   - botocore>=1.11
   - dask
   - fastparquet>=0.4.0
-  - fsspec>=0.7.4, <2021.6.0
+  - fsspec>=0.7.4
   - gcsfs>=0.6.0
   - geopandas
   - html5lib

ci/deps/actions-38-slow.yaml

@@ -13,7 +13,7 @@ dependencies:
 
   # pandas dependencies
   - beautifulsoup4
-  - fsspec>=0.7.4, <2021.6.0
+  - fsspec>=0.7.4
   - html5lib
   - lxml
   - matplotlib

ci/deps/actions-39-slow.yaml

@@ -15,7 +15,7 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
-  - fsspec>=0.8.0, <2021.6.0
+  - fsspec>=0.8.0
   - gcsfs
   - html5lib
   - jinja2

ci/deps/actions-39.yaml

@@ -14,7 +14,7 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
-  - fsspec>=0.8.0, <2021.6.0
+  - fsspec>=0.8.0
   - gcsfs
   - html5lib
   - jinja2

ci/deps/azure-windows-38.yaml

@@ -17,7 +17,7 @@ dependencies:
   - bottleneck
   - fastparquet>=0.4.0
   - flask
-  - fsspec>=0.8.0, <2021.6.0
+  - fsspec>=0.8.0
   - matplotlib=3.3.2
   - moto>=1.3.14
   - numba

ci/deps/azure-windows-39.yaml

@@ -15,7 +15,7 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4
   - bottleneck
-  - fsspec>=0.8.0, <2021.6.0
+  - fsspec>=0.8.0
   - gcsfs
   - html5lib
   - jinja2

doc/source/development/contributing_codebase.rst

@@ -395,12 +395,19 @@ This module will ultimately house types for repeatedly used concepts like "path-
 Validating type hints
 ~~~~~~~~~~~~~~~~~~~~~
 
-pandas uses `mypy <http://mypy-lang.org>`_ to statically analyze the code base and type hints. After making any change you can ensure your type hints are correct by running
+pandas uses `mypy <http://mypy-lang.org>`_ and `pyright <https://github.com/microsoft/pyright>`_ to statically analyze the code base and type hints. After making any change you can ensure your type hints are correct by running
 
 .. code-block:: shell
 
    mypy pandas
 
+   # let pre-commit setup and run pyright
+   pre-commit run --hook-stage manual --all-files pyright
+   # or if pyright is installed (requires node.js)
+   pyright
+
+A recent version of ``numpy`` (>=1.21.0) is required for type validation.
+
 .. _contributing.ci:
 
 Testing with continuous integration

doc/source/index.rst.template

@@ -12,6 +12,9 @@ pandas documentation
 
 **Download documentation**: `PDF Version <pandas.pdf>`__ | `Zipped HTML <pandas.zip>`__
 
+**Previous versions**: Documentation of previous pandas versions is available at
+`pandas.pydata.org <https://pandas.pydata.org/>`__.
+
 **Useful links**:
 `Binary Installers <https://pypi.org/project/pandas>`__ |
 `Source Repository <https://github.com/pandas-dev/pandas>`__ |

doc/source/user_guide/basics.rst

@@ -1045,6 +1045,9 @@ not noted for a particular column will be ``NaN``:
 Mixed dtypes
 ++++++++++++
 
+.. deprecated:: 1.4.0
+   Attempting to determine which columns cannot be aggregated and silently dropping them from the results is deprecated and will be removed in a future version. If any porition of the columns or operations provided fail, the call to ``.agg`` will raise.
+
 When presented with mixed dtypes that cannot aggregate, ``.agg`` will only take the valid
 aggregations. This is similar to how ``.groupby.agg`` works.
 
@@ -1061,6 +1064,7 @@ aggregations. This is similar to how ``.groupby.agg`` works.
    mdf.dtypes
 
 .. ipython:: python
+   :okwarning:
 
    mdf.agg(["min", "sum"])
 

doc/source/user_guide/groupby.rst

@@ -578,7 +578,7 @@ column, which produces an aggregated result with a hierarchical index:
 
 .. ipython:: python
 
-   grouped.agg([np.sum, np.mean, np.std])
+   grouped[["C", "D"]].agg([np.sum, np.mean, np.std])
 
 
 The resulting aggregations are named for the functions themselves. If you
@@ -597,7 +597,7 @@ For a grouped ``DataFrame``, you can rename in a similar manner:
 .. ipython:: python
 
    (
-       grouped.agg([np.sum, np.mean, np.std]).rename(
+       grouped[["C", "D"]].agg([np.sum, np.mean, np.std]).rename(
            columns={"sum": "foo", "mean": "bar", "std": "baz"}
        )
    )

doc/source/user_guide/merging.rst

@@ -562,7 +562,7 @@ all standard database join operations between ``DataFrame`` or named ``Series``
   (hierarchical), the number of levels must match the number of join keys
   from the right DataFrame or Series.
 * ``right_index``: Same usage as ``left_index`` for the right DataFrame or Series
-* ``how``: One of ``'left'``, ``'right'``, ``'outer'``, ``'inner'``. Defaults
+* ``how``: One of ``'left'``, ``'right'``, ``'outer'``, ``'inner'``, ``'cross'``. Defaults
   to ``inner``. See below for more detailed description of each method.
 * ``sort``: Sort the result DataFrame by the join keys in lexicographical
   order. Defaults to ``True``, setting to ``False`` will improve performance
@@ -707,6 +707,7 @@ either the left or right tables, the values in the joined table will be
     ``right``, ``RIGHT OUTER JOIN``, Use keys from right frame only
     ``outer``, ``FULL OUTER JOIN``, Use union of keys from both frames
     ``inner``, ``INNER JOIN``, Use intersection of keys from both frames
+    ``cross``, ``CROSS JOIN``, Create the cartesian product of rows of both frames
 
 .. ipython:: python
 
@@ -751,6 +752,17 @@ either the left or right tables, the values in the joined table will be
    p.plot([left, right], result, labels=["left", "right"], vertical=False);
    plt.close("all");
 
+.. ipython:: python
+
+   result = pd.merge(left, right, how="cross")
+
+.. ipython:: python
+   :suppress:
+
+   @savefig merging_merge_cross.png
+   p.plot([left, right], result, labels=["left", "right"], vertical=False);
+   plt.close("all");
+
 You can merge a mult-indexed Series and a DataFrame, if the names of
 the MultiIndex correspond to the columns from the DataFrame. Transform
 the Series to a DataFrame using :meth:`Series.reset_index` before merging,

doc/source/user_guide/visualization.rst

@@ -1651,7 +1651,7 @@ remedy this, ``DataFrame`` plotting supports the use of the ``colormap`` argumen
 which accepts either a Matplotlib `colormap <https://matplotlib.org/api/cm_api.html>`__
 or a string that is a name of a colormap registered with Matplotlib. A
 visualization of the default matplotlib colormaps is available `here
-<https://matplotlib.org/examples/color/colormaps_reference.html>`__.
+<https://matplotlib.org/stable/gallery/color/colormap_reference.html>`__.
 
 As matplotlib does not directly support colormaps for line-based plots, the
 colors are selected based on an even spacing determined by the number of columns

doc/source/whatsnew/v0.20.0.rst

@@ -105,6 +105,7 @@ aggregations. This is similar to how groupby ``.agg()`` works. (:issue:`15015`)
    df.dtypes
 
 .. ipython:: python
+   :okwarning:
 
    df.agg(['min', 'sum'])
 

doc/source/whatsnew/v1.3.4.rst

@@ -14,14 +14,18 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
+- Fixed regression in :meth:`.GroupBy.agg` where it was failing silently with mixed data types along ``axis=1`` and :class:`MultiIndex` (:issue:`43209`)
 - Fixed regression in :meth:`merge` with integer and ``NaN`` keys failing with ``outer`` merge (:issue:`43550`)
 - Fixed regression in :meth:`DataFrame.corr` raising ``ValueError`` with ``method="spearman"`` on 32-bit platforms (:issue:`43588`)
 - Fixed performance regression in :meth:`MultiIndex.equals` (:issue:`43549`)
 - Fixed performance regression in :meth:`.GroupBy.first` and :meth:`.GroupBy.last` with :class:`StringDtype` (:issue:`41596`)
 - Fixed regression in :meth:`Series.cat.reorder_categories` failing to update the categories on the ``Series`` (:issue:`43232`)
 - Fixed regression in :meth:`Series.cat.categories` setter failing to update the categories on the ``Series`` (:issue:`43334`)
 - Fixed regression in :meth:`pandas.read_csv` raising ``UnicodeDecodeError`` exception when ``memory_map=True`` (:issue:`43540`)
+- Fixed regression in :meth:`DataFrame.explode` raising ``AssertionError`` when ``column`` is any scalar which is not a string (:issue:`43314`)
 - Fixed regression in :meth:`Series.aggregate` attempting to pass ``args`` and ``kwargs`` multiple times to the user supplied ``func`` in certain cases (:issue:`43357`)
+- Fixed regression when iterating over a :class:`DataFrame.groupby.rolling` object causing the resulting DataFrames to have an incorrect index if the input groupings were not sorted (:issue:`43386`)
+- Fixed regression in :meth:`DataFrame.groupby.rolling.cov` and :meth:`DataFrame.groupby.rolling.corr` computing incorrect results if the input groupings were not sorted (:issue:`43386`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.4.0.rst

@@ -93,7 +93,7 @@ Multithreaded CSV reading with a new CSV Engine based on pyarrow
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 :func:`pandas.read_csv` now accepts ``engine="pyarrow"`` (requires at least ``pyarrow`` 0.17.0) as an argument, allowing for faster csv parsing on multicore machines
-with pyarrow installed. See the :doc:`I/O docs </user_guide/io>` for more info. (:issue:`23697`)
+with pyarrow installed. See the :doc:`I/O docs </user_guide/io>` for more info. (:issue:`23697`, :issue:`43706`)
 
 .. _whatsnew_140.enhancements.window_rank:
 
@@ -336,7 +336,10 @@ Other Deprecations
 - Deprecated the 'include_start' and 'include_end' arguments in :meth:`DataFrame.between_time`; in a future version passing 'include_start' or 'include_end' will raise (:issue:`40245`)
 - Deprecated the ``squeeze`` argument to :meth:`read_csv`, :meth:`read_table`, and :meth:`read_excel`. Users should squeeze the DataFrame afterwards with ``.squeeze("columns")`` instead. (:issue:`43242`)
 - Deprecated the ``index`` argument to :class:`SparseArray` construction (:issue:`23089`)
+- Deprecated the ``closed`` argument in :meth:`date_range` and :meth:`bdate_range` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated :meth:`.Rolling.validate`, :meth:`.Expanding.validate`, and :meth:`.ExponentialMovingWindow.validate` (:issue:`43665`)
+- Deprecated silent dropping of columns that raised a ``TypeError`` in :class:`Series.transform` and :class:`DataFrame.transform` when used with a dictionary (:issue:`43740`)
+- Deprecated silent dropping of columns that raised a ``TypeError``, ``DataError``, and some cases of ``ValueError`` in :meth:`Series.aggregate`, :meth:`DataFrame.aggregate`, :meth:`Series.groupby.aggregate`, and :meth:`DataFrame.groupby.aggregate` when used with a list (:issue:`43740`)
 
 .. ---------------------------------------------------------------------------
 
@@ -345,6 +348,7 @@ Other Deprecations
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 - Performance improvement in :meth:`.GroupBy.sample`, especially when ``weights`` argument provided (:issue:`34483`)
+- Performance improvement when converting non-string arrays to string arrays (:issue:`34483`)
 - Performance improvement in :meth:`.GroupBy.transform` for user-defined functions (:issue:`41598`)
 - Performance improvement in constructing :class:`DataFrame` objects (:issue:`42631`)
 - Performance improvement in :meth:`GroupBy.shift` when ``fill_value`` argument is provided (:issue:`26615`)
@@ -356,8 +360,10 @@ Performance improvements
 - Performance improvement in indexing with a :class:`MultiIndex` indexer on another :class:`MultiIndex` (:issue:43370`)
 - Performance improvement in :meth:`GroupBy.quantile` (:issue:`43469`)
 - :meth:`SparseArray.min` and :meth:`SparseArray.max` no longer require converting to a dense array (:issue:`43526`)
+- Indexing into a :class:`SparseArray` with a ``slice`` with ``step=1`` no longer requires converting to a dense array (:issue:`43777`)
 - Performance improvement in :meth:`SparseArray.take` with ``allow_fill=False`` (:issue:`43654`)
 - Performance improvement in :meth:`.Rolling.mean` and :meth:`.Expanding.mean` with ``engine="numba"`` (:issue:`43612`)
+-
 
 .. ---------------------------------------------------------------------------
 
@@ -380,6 +386,7 @@ Datetimelike
 - Bug in :class:`DataFrame` constructor unnecessarily copying non-datetimelike 2D object arrays (:issue:`39272`)
 - Bug in :func:`to_datetime` with ``format`` and ``pandas.NA`` was raising ``ValueError`` (:issue:`42957`)
 - :func:`to_datetime` would silently swap ``MM/DD/YYYY`` and ``DD/MM/YYYY`` formats if the given ``dayfirst`` option could not be respected - now, a warning is raised in the case of delimited date strings (e.g. ``31-12-2012``) (:issue:`12585`)
+- Bug in :meth:`date_range` and :meth:`bdate_range` do not return right bound when ``start`` = ``end`` and set is closed on one side (:issue:`43394`)
 -
 
 Timedelta
@@ -417,6 +424,7 @@ Interval
 
 Indexing
 ^^^^^^^^
+- Bug in :meth:`Series.rename` when index in Series is MultiIndex and level in rename is provided. (:issue:`43659`)
 - Bug in :meth:`DataFrame.truncate` and :meth:`Series.truncate` when the object's Index has a length greater than one but only one unique value (:issue:`42365`)
 - Bug in :meth:`Series.loc` and :meth:`DataFrame.loc` with a :class:`MultiIndex` when indexing with a tuple in which one of the levels is also a tuple (:issue:`27591`)
 - Bug in :meth:`Series.loc` when with a :class:`MultiIndex` whose first level contains only ``np.nan`` values (:issue:`42055`)
@@ -429,7 +437,8 @@ Indexing
 - Bug in :meth:`DataFrame.drop` where the error message did not show missing labels with commas when raising ``KeyError`` (:issue:`42881`)
 - Bug in :meth:`DataFrame.query` where method calls in query strings led to errors when the ``numexpr`` package was installed. (:issue:`22435`)
 - Bug in :meth:`DataFrame.nlargest` and :meth:`Series.nlargest` where sorted result did not count indexes containing ``np.nan`` (:issue:`28984`)
-
+- Bug in indexing on a non-unique object-dtype :class:`Index` with an NA scalar (e.g. ``np.nan``) (:issue:`43711`)
+-
 
 Missing
 ^^^^^^^
@@ -456,6 +465,9 @@ I/O
 - Bug in unpickling a :class:`Index` with object dtype incorrectly inferring numeric dtypes (:issue:`43188`)
 - Bug in :func:`read_csv` where reading multi-header input with unequal lengths incorrectly raising uncontrolled ``IndexError`` (:issue:`43102`)
 - Bug in :func:`read_csv`, changed exception class when expecting a file path name or file-like object from ``OSError`` to ``TypeError`` (:issue:`43366`)
+- Bug in :func:`json_normalize` where multi-character ``sep`` parameter is incorrectly prefixed to every key (:issue:`43831`)
+- Bug in :func:`read_csv` with :code:`float_precision="round_trip"` which did not skip initial/trailing whitespace (:issue:`43713`)
+-
 
 Period
 ^^^^^^
@@ -482,6 +494,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrame.rolling.corr` when the :class:`DataFrame` columns was a :class:`MultiIndex` (:issue:`21157`)
 - Bug in :meth:`DataFrame.groupby.rolling` when specifying ``on`` and calling ``__getitem__`` would subsequently return incorrect results (:issue:`43355`)
 - Bug in :meth:`GroupBy.apply` with time-based :class:`Grouper` objects incorrectly raising ``ValueError`` in corner cases where the grouping vector contains a ``NaT`` (:issue:`43500`, :issue:`43515`)
+- Bug in :meth:`GroupBy.mean` failing with ``complex`` dtype (:issue:`43701`)
 
 Reshaping
 ^^^^^^^^^
@@ -497,7 +510,7 @@ Sparse
 ^^^^^^
 - Bug in :meth:`DataFrame.sparse.to_coo` raising ``AttributeError`` when column names are not unique (:issue:`29564`)
 - Bug in :meth:`SparseArray.max` and :meth:`SparseArray.min` raising ``ValueError`` for arrays with 0 non-null elements (:issue:`43527`)
--
+- Bug in :meth:`DataFrame.sparse.to_coo` silently converting non-zero fill values to zero (:issue:`24817`)
 -
 
 ExtensionArray