Merge branch 'master' into 42452

Author: debnathshoham

asv_bench/benchmarks/groupby.py

@@ -369,6 +369,18 @@ def time_category_size(self):
         self.draws.groupby(self.cats).size()
 
 
+class Shift:
+    def setup(self):
+        N = 18
+        self.df = DataFrame({"g": ["a", "b"] * 9, "v": list(range(N))})
+
+    def time_defaults(self):
+        self.df.groupby("g").shift()
+
+    def time_fill_value(self):
+        self.df.groupby("g").shift(fill_value=99)
+
+
 class FillNA:
     def setup(self):
         N = 100

asv_bench/benchmarks/reshape.py

@@ -102,6 +102,7 @@ def setup(self, dtype):
         columns = np.arange(n)
         if dtype == "int":
             values = np.arange(m * m * n).reshape(m * m, n)
+            self.df = DataFrame(values, index, columns)
         else:
             # the category branch is ~20x slower than int. So we
             # cut down the size a bit. Now it's only ~3x slower.
@@ -111,7 +112,10 @@ def setup(self, dtype):
             values = np.take(list(string.ascii_letters), indices)
             values = [pd.Categorical(v) for v in values.T]
 
-        self.df = DataFrame(values, index, columns)
+            self.df = DataFrame(
+                {i: cat for i, cat in enumerate(values)}, index, columns
+            )
+
         self.df2 = self.df.iloc[:-1]
 
     def time_full_product(self, dtype):

ci/code_checks.sh

@@ -107,44 +107,22 @@ fi
 ### DOCTESTS ###
 if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
-    MSG='Doctests for individual files' ; echo $MSG
-    pytest -q --doctest-modules \
-      pandas/core/accessor.py \
-      pandas/core/aggregation.py \
-      pandas/core/algorithms.py \
-      pandas/core/base.py \
-      pandas/core/construction.py \
-      pandas/core/frame.py \
-      pandas/core/generic.py \
-      pandas/core/indexers.py \
-      pandas/core/nanops.py \
-      pandas/core/series.py \
-      pandas/io/sql.py
-    RET=$(($RET + $?)) ; echo $MSG "DONE"
-
-    MSG='Doctests for directories' ; echo $MSG
-    pytest -q --doctest-modules \
+    MSG='Doctests' ; echo $MSG
+    python -m pytest --doctest-modules \
       pandas/_libs/ \
       pandas/api/ \
       pandas/arrays/ \
       pandas/compat/ \
-      pandas/core/array_algos/ \
-      pandas/core/arrays/ \
-      pandas/core/computation/ \
-      pandas/core/dtypes/ \
-      pandas/core/groupby/ \
-      pandas/core/indexes/ \
-      pandas/core/ops/ \
-      pandas/core/reshape/ \
-      pandas/core/strings/ \
-      pandas/core/tools/ \
-      pandas/core/window/ \
+      pandas/core \
       pandas/errors/ \
       pandas/io/clipboard/ \
       pandas/io/json/ \
       pandas/io/excel/ \
       pandas/io/parsers/ \
       pandas/io/sas/ \
+      pandas/io/sql.py \
+      pandas/io/formats/format.py \
+      pandas/io/formats/style.py \
       pandas/tseries/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 

ci/deps/actions-39-slow.yaml

@@ -23,6 +23,7 @@ dependencies:
   - matplotlib
   - moto>=1.3.14
   - flask
+  - numba
   - numexpr
   - numpy
   - openpyxl

ci/deps/actions-39.yaml

@@ -22,6 +22,7 @@ dependencies:
   - matplotlib
   - moto>=1.3.14
   - flask
+  - numba
   - numexpr
   - numpy
   - openpyxl

ci/deps/azure-windows-39.yaml

@@ -23,6 +23,7 @@ dependencies:
   - matplotlib
   - moto>=1.3.14
   - flask
+  - numba
   - numexpr
   - numpy
   - openpyxl

ci/run_tests.sh

@@ -30,7 +30,10 @@ fi
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"
 
-PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS  --dist=loadfile $TEST_ARGS $COVERAGE pandas"
+if [[ "$PANDAS_DATA_MANAGER" != "array" ]]; then
+    # The ArrayManager tests should have already been run by PYTEST_CMD if PANDAS_DATA_MANAGER was already set to array
+    PYTEST_AM_CMD="PANDAS_DATA_MANAGER=array pytest -m \"$PATTERN and arraymanager\" -n $PYTEST_WORKERS  --dist=loadfile $TEST_ARGS $COVERAGE pandas"
 
-echo $PYTEST_AM_CMD
-sh -c "$PYTEST_AM_CMD"
+    echo $PYTEST_AM_CMD
+    sh -c "$PYTEST_AM_CMD"
+fi

doc/source/_static/style/df_pipe.png

@@ -12,6 +12,11 @@
 Nullable Boolean data type
 **************************
 
+.. note::
+
+   BooleanArray is currently experimental. Its API or implementation may
+   change without warning.
+
 .. versionadded:: 1.0.0
 
 

doc/source/user_guide/boolean.rst

@@ -1300,7 +1300,7 @@ is closed.
 
 .. ipython:: python
 
-   store = pd.HDFStore("test.h5", "w", diver="H5FD_CORE")
+   store = pd.HDFStore("test.h5", "w", driver="H5FD_CORE")
 
    df = pd.DataFrame(np.random.randn(8, 3))
    store["test"] = df

doc/source/user_guide/cookbook.rst

@@ -316,6 +316,34 @@ The ``by`` keyword can be specified to plot grouped histograms:
    @savefig grouped_hist.png
    data.hist(by=np.random.randint(0, 4, 1000), figsize=(6, 4));
 
+.. ipython:: python
+   :suppress:
+
+   plt.close("all")
+   np.random.seed(123456)
+
+In addition, the ``by`` keyword can also be specified in :meth:`DataFrame.plot.hist`.
+
+.. versionchanged:: 1.4.0
+
+.. ipython:: python
+
+   data = pd.DataFrame(
+       {
+           "a": np.random.choice(["x", "y", "z"], 1000),
+           "b": np.random.choice(["e", "f", "g"], 1000),
+           "c": np.random.randn(1000),
+           "d": np.random.randn(1000) - 1,
+       },
+   )
+
+   @savefig grouped_hist_by.png
+   data.plot.hist(by=["a", "b"], figsize=(10, 5));
+
+.. ipython:: python
+   :suppress:
+
+   plt.close("all")
 
 .. _visualization.box:
 
@@ -448,6 +476,32 @@ columns:
 
     plt.close("all")
 
+You could also create groupings with :meth:`DataFrame.plot.box`, for instance:
+
+.. versionchanged:: 1.4.0
+
+.. ipython:: python
+   :suppress:
+
+   plt.close("all")
+   np.random.seed(123456)
+
+.. ipython:: python
+   :okwarning:
+
+   df = pd.DataFrame(np.random.rand(10, 3), columns=["Col1", "Col2", "Col3"])
+   df["X"] = pd.Series(["A", "A", "A", "A", "A", "B", "B", "B", "B", "B"])
+
+   plt.figure();
+
+   @savefig box_plot_ex4.png
+   bp = df.plot.box(column=["Col1", "Col2"], by="X")
+
+.. ipython:: python
+   :suppress:
+
+    plt.close("all")
+
 .. _visualization.box.return:
 
 In ``boxplot``, the return type can be controlled by the ``return_type``, keyword. The valid choices are ``{"axes", "dict", "both", None}``.

doc/source/user_guide/visualization.rst

@@ -24,6 +24,7 @@ Version 1.3
 .. toctree::
    :maxdepth: 2
 
+   v1.3.2
    v1.3.1
    v1.3.0
 

doc/source/whatsnew/index.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_131:
 
-What's new in 1.3.1 (July ??, 2021)
+What's new in 1.3.1 (July 25, 2021)
 -----------------------------------
 
 These are the changes in pandas 1.3.1. See :ref:`release` for a full changelog
@@ -25,6 +25,10 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrame.isin` and :meth:`Series.isin` raising ``TypeError`` with nullable data containing at least one missing value (:issue:`42405`)
 - Regression in :func:`concat` between objects with bool dtype and integer dtype casting to object instead of to integer (:issue:`42092`)
 - Bug in :class:`Series` constructor not accepting a ``dask.Array`` (:issue:`38645`)
+- Fixed regression for ``SettingWithCopyWarning`` displaying incorrect stacklevel (:issue:`42570`)
+- Fixed regression for :func:`merge_asof` raising ``KeyError`` when one of the ``by`` columns is in the index (:issue:`34488`)
+- Fixed regression in :func:`to_datetime` returning pd.NaT for inputs that produce duplicated values, when ``cache=True`` (:issue:`42259`)
+- Fixed regression in :meth:`SeriesGroupBy.value_counts` that resulted in an ``IndexError`` when called on a Series with one row (:issue:`42618`)
 
 .. ---------------------------------------------------------------------------
 
@@ -34,16 +38,8 @@ Bug fixes
 ~~~~~~~~~
 - Fixed bug in :meth:`DataFrame.transpose` dropping values when the DataFrame had an Extension Array dtype and a duplicate index (:issue:`42380`)
 - Fixed bug in :meth:`DataFrame.to_xml` raising ``KeyError`` when called with ``index=False`` and an offset index (:issue:`42458`)
--
-
-.. ---------------------------------------------------------------------------
-
-.. _whatsnew_131.other:
-
-Other
-~~~~~
--
--
+- Fixed bug in :meth:`.Styler.set_sticky` not handling index names correctly for single index columns case (:issue:`42537`)
+- Fixed bug in :meth:`DataFrame.copy` failing to consolidate blocks in the result (:issue:`42579`)
 
 .. ---------------------------------------------------------------------------
 
@@ -52,4 +48,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v1.3.0..v1.3.1|HEAD
+.. contributors:: v1.3.0..v1.3.1

doc/source/whatsnew/v1.3.1.rst

@@ -0,0 +1,52 @@
+.. _whatsnew_132:
+
+What's new in 1.3.2 (August ??, 2021)
+-------------------------------------
+
+These are the changes in pandas 1.3.2. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_132.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+- Performance regression in :meth:`DataFrame.isin` and :meth:`Series.isin` for nullable data types (:issue:`42714`)
+- Regression in updating values of :class:`pandas.Series` using boolean index, created by using :meth:`pandas.DataFrame.pop` (:issue:`42530`)
+- Regression in :meth:`DataFrame.from_records` with empty records (:issue:`42456`)
+- Fixed regression in :meth:`DataFrame.shift` where TypeError occurred when shifting DataFrame created by concatenation of slices and fills with values (:issue:`42719`)
+- Regression in :meth:`DataFrame.agg` when the ``func`` argument returned lists and ``axis=1`` (:issue:`42727`)
+- Regression in :meth:`DataFrame.drop` does nothing if :class:`MultiIndex` has duplicates and indexer is a tuple or list of tuples (:issue:`42771`)
+- Fixed regression where :meth:`pandas.read_csv` raised a ``ValueError`` when parameters ``names`` and ``prefix`` were both set to None (:issue:`42387`)
+- Fixed regression in comparisons between :class:`Timestamp` object and ``datetime64`` objects outside the implementation bounds for nanosecond ``datetime64`` (:issue:`42794`)
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_132.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+- 1D slices over extension types turn into N-dimensional slices over ExtensionArrays (:issue:`42430`)
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_132.other:
+
+Other
+~~~~~
+-
+-
+
+.. ---------------------------------------------------------------------------
+
+.. _whatsnew_132.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v1.3.1..v1.3.2|HEAD