Merge branch 'pandas-dev:main' into raise-on-parse-int-overflow

Author: SandroCasagrande

.circleci/config.yml

@@ -4,7 +4,7 @@ jobs:
   test-arm:
     machine:
       image: ubuntu-2004:202101-01
-    resource_class: arm.medium
+    resource_class: arm.large
     environment:
       ENV_FILE: ci/deps/circle-38-arm64.yaml
       PYTEST_WORKERS: auto

ci/deps/actions-310.yaml

@@ -19,6 +19,7 @@ dependencies:
   - pytz
 
   # optional dependencies
+  - aiobotocore<2.0.0
   - beautifulsoup4
   - blosc
   - bottleneck
@@ -43,7 +44,7 @@ dependencies:
   - pyreadstat
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.05.0
   - scipy
   - sqlalchemy
   - tabulate

ci/deps/actions-38-downstream_compat.yaml

@@ -44,7 +44,7 @@ dependencies:
   - pytables
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.05.0
   - scipy
   - sqlalchemy
   - tabulate

ci/deps/actions-38.yaml

@@ -19,6 +19,7 @@ dependencies:
   - pytz
 
   # optional dependencies
+  - aiobotocore<2.0.0
   - beautifulsoup4
   - blosc
   - bottleneck
@@ -43,7 +44,7 @@ dependencies:
   - pytables
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.05.0
   - scipy
   - sqlalchemy
   - tabulate

ci/deps/actions-39.yaml

@@ -19,6 +19,7 @@ dependencies:
   - pytz
 
   # optional dependencies
+  - aiobotocore<2.0.0
   - beautifulsoup4
   - blosc
   - bottleneck
@@ -43,7 +44,7 @@ dependencies:
   - pytables
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.05.0
   - scipy
   - sqlalchemy
   - tabulate

ci/deps/circle-38-arm64.yaml

@@ -19,6 +19,7 @@ dependencies:
   - pytz
 
   # optional dependencies
+  - aiobotocore<2.0.0
   - beautifulsoup4
   - blosc
   - bottleneck
@@ -44,7 +45,7 @@ dependencies:
   - pytables
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.05.0
   - scipy
   - sqlalchemy
   - tabulate

doc/source/development/contributing_codebase.rst

@@ -265,7 +265,11 @@ pandas uses `mypy <http://mypy-lang.org>`_ and `pyright <https://github.com/micr
 
 .. code-block:: shell
 
-   pre-commit run --hook-stage manual --all-files
+    # the following might fail if the installed pandas version does not correspond to your local git version
+    pre-commit run --hook-stage manual --all-files
+
+    # if the above fails due to stubtest
+    SKIP=stubtest pre-commit run --hook-stage manual --all-files
 
 in your activated python environment. A recent version of ``numpy`` (>=1.22.0) is required for type validation.
 

doc/source/getting_started/intro_tutorials/05_add_columns.rst

@@ -30,8 +30,8 @@
     </ul>
     </div>
 
-How to create new columns derived from existing columns?
---------------------------------------------------------
+How to create new columns derived from existing columns
+-------------------------------------------------------
 
 .. image:: ../../_static/schemas/05_newcolumn_1.svg
    :align: center

doc/source/getting_started/intro_tutorials/06_calculate_statistics.rst

@@ -30,8 +30,8 @@
         </ul>
     </div>
 
-How to calculate summary statistics?
-------------------------------------
+How to calculate summary statistics
+-----------------------------------
 
 Aggregating statistics
 ~~~~~~~~~~~~~~~~~~~~~~

doc/source/getting_started/intro_tutorials/07_reshape_table_layout.rst

@@ -84,8 +84,8 @@ measurement.
         </ul>
     </div>
 
-How to reshape the layout of tables?
-------------------------------------
+How to reshape the layout of tables
+-----------------------------------
 
 Sort table rows
 ~~~~~~~~~~~~~~~

doc/source/getting_started/intro_tutorials/08_combine_dataframes.rst

@@ -88,8 +88,8 @@ Westminster* in respectively Paris, Antwerp and London.
     </div>
 
 
-How to combine data from multiple tables?
------------------------------------------
+How to combine data from multiple tables
+----------------------------------------
 
 Concatenating objects
 ~~~~~~~~~~~~~~~~~~~~~

doc/source/getting_started/intro_tutorials/09_timeseries.rst

@@ -55,8 +55,8 @@ Westminster* in respectively Paris, Antwerp and London.
     </ul>
     </div>
 
-How to handle time series data with ease?
------------------------------------------
+How to handle time series data with ease
+----------------------------------------
 
 .. _10min_tut_09_timeseries.properties:
 

doc/source/getting_started/intro_tutorials/10_text_data.rst

@@ -29,8 +29,8 @@
         </ul>
     </div>
 
-How to manipulate textual data?
--------------------------------
+How to manipulate textual data
+------------------------------
 
 .. raw:: html
 

doc/source/index.rst.template

@@ -10,7 +10,7 @@ pandas documentation
 
 **Date**: |today| **Version**: |version|
 
-**Download documentation**: `PDF Version <pandas.pdf>`__ | `Zipped HTML <pandas.zip>`__
+**Download documentation**: `Zipped HTML <pandas.zip>`__
 
 **Previous versions**: Documentation of previous pandas versions is available at
 `pandas.pydata.org <https://pandas.pydata.org/>`__.

doc/source/whatsnew/v1.4.4.rst

@@ -26,6 +26,7 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrame.loc` setting a length-1 array like value to a single value in the DataFrame (:issue:`46268`)
 - Fixed regression when slicing with :meth:`DataFrame.loc` with :class:`DateOffset`-index (:issue:`46671`)
 - Fixed regression in setting ``None`` or non-string value into a ``string``-dtype Series using a mask (:issue:`47628`)
+- Fixed regression in updating a DataFrame column through Series ``__setitem__`` (using chained assignment) not updating column values inplace and using too much memory (:issue:`47172`)
 - Fixed regression in :meth:`DataFrame.select_dtypes` returning a view on the original DataFrame (:issue:`48090`)
 - Fixed regression using custom Index subclasses (for example, used in xarray) with :meth:`~DataFrame.reset_index` or :meth:`Index.insert` (:issue:`47071`)
 - Fixed regression in :meth:`DatetimeIndex.intersection` when the :class:`DatetimeIndex` has dates crossing daylight savings time (:issue:`46702`)

environment.yml

@@ -45,7 +45,7 @@ dependencies:
   - pytables
   - python-snappy
   - pyxlsb
-  - s3fs
+  - s3fs>=2021.05.0
   - scipy
   - sqlalchemy
   - tabulate

pandas/core/reshape/merge.py

@@ -242,7 +242,7 @@ def merge_ordered(
     Returns
     -------
     DataFrame
-        The merged DataFrame output type will the be same as
+        The merged DataFrame output type will be the same as
         'left', if it is a subclass of DataFrame.
 
     See Also

pandas/core/series.py

@@ -1169,7 +1169,7 @@ def __setitem__(self, key, value) -> None:
                 self._set_with(key, value)
 
         if cacher_needs_updating:
-            self._maybe_update_cacher()
+            self._maybe_update_cacher(inplace=True)
 
     def _set_with_engine(self, key, value) -> None:
         loc = self.index.get_loc(key)

pandas/tests/arrays/floating/test_astype.py

@@ -116,3 +116,13 @@ def test_astype_object(dtype):
     # check exact element types
     assert isinstance(result[0], float)
     assert result[1] is pd.NA
+
+
+def test_Float64_conversion():
+    # GH#40729
+    testseries = pd.Series(["1", "2", "3", "4"], dtype="object")
+    result = testseries.astype(pd.Float64Dtype())
+
+    expected = pd.Series([1.0, 2.0, 3.0, 4.0], dtype=pd.Float64Dtype())
+
+    tm.assert_series_equal(result, expected)

pandas/tests/arrays/test_timedeltas.py

@@ -67,6 +67,13 @@ def test_total_seconds(self, unit, tda):
         expected = tda_nano.total_seconds()
         tm.assert_numpy_array_equal(result, expected)
 
+    def test_timedelta_array_total_seconds(self):
+        # GH34290
+        expected = Timedelta("2 min").total_seconds()
+
+        result = pd.array([Timedelta("2 min")]).total_seconds()[0]
+        assert result == expected
+
     @pytest.mark.parametrize(
         "nat", [np.datetime64("NaT", "ns"), np.datetime64("NaT", "us")]
     )

pandas/tests/extension/test_floating.py

@@ -211,5 +211,6 @@ class TestParsing(base.BaseParsingTests):
     pass
 
 
+@pytest.mark.filterwarnings("ignore:overflow encountered in reduce:RuntimeWarning")
 class Test2DCompat(base.Dim2CompatTests):
     pass

pandas/tests/frame/indexing/test_setitem.py

@@ -1235,3 +1235,21 @@ def test_setitem_not_operating_inplace(self, value, set_value, indexer):
         view = df[:]
         df[indexer] = set_value
         tm.assert_frame_equal(view, expected)
+
+    @td.skip_array_manager_invalid_test
+    def test_setitem_column_update_inplace(self, using_copy_on_write):
+        # https://github.com/pandas-dev/pandas/issues/47172
+
+        labels = [f"c{i}" for i in range(10)]
+        df = DataFrame({col: np.zeros(len(labels)) for col in labels}, index=labels)
+        values = df._mgr.blocks[0].values
+
+        for label in df.columns:
+            df[label][label] = 1
+
+        if not using_copy_on_write:
+            # diagonal values all updated
+            assert np.all(values[np.arange(10), np.arange(10)] == 1)
+        else:
+            # original dataframe not updated
+            assert np.all(values[np.arange(10), np.arange(10)] == 0)

pandas/tests/frame/methods/test_combine_first.py

@@ -3,6 +3,9 @@
 import numpy as np
 import pytest
 
+from pandas.compat import pa_version_under7p0
+from pandas.errors import PerformanceWarning
+
 from pandas.core.dtypes.cast import (
     find_common_type,
     is_dtype_equal,
@@ -387,12 +390,24 @@ def test_combine_first_string_dtype_only_na(self, nullable_string_dtype):
             {"a": ["962", "85"], "b": [pd.NA] * 2}, dtype=nullable_string_dtype
         )
         df2 = DataFrame({"a": ["85"], "b": [pd.NA]}, dtype=nullable_string_dtype)
-        df = df.set_index(["a", "b"], copy=False)
-        df2 = df2.set_index(["a", "b"], copy=False)
+        with tm.maybe_produces_warning(
+            PerformanceWarning,
+            pa_version_under7p0 and nullable_string_dtype == "string[pyarrow]",
+        ):
+            df = df.set_index(["a", "b"], copy=False)
+        with tm.maybe_produces_warning(
+            PerformanceWarning,
+            pa_version_under7p0 and nullable_string_dtype == "string[pyarrow]",
+        ):
+            df2 = df2.set_index(["a", "b"], copy=False)
         result = df.combine_first(df2)
-        expected = DataFrame(
-            {"a": ["962", "85"], "b": [pd.NA] * 2}, dtype=nullable_string_dtype
-        ).set_index(["a", "b"])
+        with tm.maybe_produces_warning(
+            PerformanceWarning,
+            pa_version_under7p0 and nullable_string_dtype == "string[pyarrow]",
+        ):
+            expected = DataFrame(
+                {"a": ["962", "85"], "b": [pd.NA] * 2}, dtype=nullable_string_dtype
+            ).set_index(["a", "b"])
         tm.assert_frame_equal(result, expected)
 
 

pandas/tests/frame/methods/test_quantile.py

@@ -751,6 +751,9 @@ def test_quantile_empty_no_rows_ints(self, interp_method):
         exp = Series([np.nan, np.nan], index=["a", "b"], name=0.5)
         tm.assert_series_equal(res, exp)
 
+    @pytest.mark.filterwarnings(
+        "ignore:The behavior of DatetimeArray._from_sequence:FutureWarning"
+    )
     def test_quantile_empty_no_rows_dt64(self, interp_method):
         interpolation, method = interp_method
         # datetimes

pandas/tests/indexes/multi/test_constructors.py

@@ -7,6 +7,8 @@
 import numpy as np
 import pytest
 
+from pandas.compat import pa_version_under1p01
+
 from pandas.core.dtypes.cast import construct_1d_object_array_from_listlike
 
 import pandas as pd
@@ -648,6 +650,28 @@ def test_from_frame():
     tm.assert_index_equal(expected, result)
 
 
+@pytest.mark.skipif(pa_version_under1p01, reason="Import Problem")
+def test_from_frame_missing_values_multiIndex():
+    # GH 39984
+    import pyarrow as pa
+
+    df = pd.DataFrame(
+        {
+            "a": Series([1, 2, None], dtype="Int64"),
+            "b": pd.Float64Dtype().__from_arrow__(pa.array([0.2, np.nan, None])),
+        }
+    )
+    multi_indexed = MultiIndex.from_frame(df)
+    expected = MultiIndex.from_arrays(
+        [
+            Series([1, 2, None]).astype("Int64"),
+            pd.Float64Dtype().__from_arrow__(pa.array([0.2, np.nan, None])),
+        ],
+        names=["a", "b"],
+    )
+    tm.assert_index_equal(multi_indexed, expected)
+
+
 @pytest.mark.parametrize(
     "non_frame",
     [

pandas/tests/scalar/timedelta/test_timedelta.py

@@ -14,7 +14,6 @@
     iNaT,
 )
 from pandas._libs.tslibs.dtypes import NpyDatetimeUnit
-from pandas.compat import IS64
 from pandas.errors import OutOfBoundsTimedelta
 
 import pandas as pd
@@ -691,7 +690,7 @@ def test_round_implementation_bounds(self):
         with pytest.raises(OverflowError, match=msg):
             Timedelta.max.ceil("s")
 
-    @pytest.mark.xfail(not IS64, reason="Failing on 32 bit build", strict=False)
+    @pytest.mark.xfail(reason="Failing on builds", strict=False)
     @given(val=st.integers(min_value=iNaT + 1, max_value=lib.i8max))
     @pytest.mark.parametrize(
         "method", [Timedelta.round, Timedelta.floor, Timedelta.ceil]

pandas/tests/scalar/timestamp/test_unary_ops.py

@@ -21,7 +21,6 @@
 )
 from pandas._libs.tslibs.dtypes import NpyDatetimeUnit
 from pandas._libs.tslibs.period import INVALID_FREQ_ERR_MSG
-from pandas.compat import IS64
 import pandas.util._test_decorators as td
 
 import pandas._testing as tm
@@ -298,7 +297,7 @@ def test_round_implementation_bounds(self):
         with pytest.raises(OverflowError, match=msg):
             Timestamp.max.ceil("s")
 
-    @pytest.mark.xfail(not IS64, reason="Failing on 32 bit build", strict=False)
+    @pytest.mark.xfail(reason="Failing on builds", strict=False)
     @given(val=st.integers(iNaT + 1, lib.i8max))
     @pytest.mark.parametrize(
         "method", [Timestamp.round, Timestamp.floor, Timestamp.ceil]

pandas/tests/test_expressions.py

@@ -173,7 +173,7 @@ def testit():
 
             with warnings.catch_warnings():
                 # array has 0s
-                msg = "invalid value encountered in true_divide"
+                msg = "invalid value encountered in divide|true_divide"
                 warnings.filterwarnings("ignore", msg, RuntimeWarning)
                 result = expr.evaluate(op, left, left, use_numexpr=True)
                 expected = expr.evaluate(op, left, left, use_numexpr=False)