Merge branch 'main' into cow_putmask

Author: phofl

.github/workflows/python-dev.yml

@@ -73,7 +73,7 @@ jobs:
       run: |
         python --version
         python -m pip install --upgrade pip setuptools wheel
-        python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
+        python -m pip install --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
         python -m pip install versioneer[toml]
         python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17

doc/scripts/eval_performance.py

@@ -6,6 +6,7 @@
 from pandas import DataFrame
 
 setup_common = """from pandas import DataFrame
+import numpy as np
 df = DataFrame(np.random.randn(%d, 3), columns=list('abc'))
 %s"""
 

doc/source/whatsnew/v1.5.3.rst

@@ -38,6 +38,13 @@ Bug fixes
 
 Other
 ~~~~~
+
+.. note::
+
+    If you are using :meth:`DataFrame.to_sql`, :func:`read_sql`, :func:`read_sql_table`, or :func:`read_sql_query` with SQLAlchemy 1.4.46 or greater,
+    you may see a ``sqlalchemy.exc.RemovedIn20Warning``. These warnings can be safely ignored for the SQLAlchemy 1.4.x releases
+    as pandas works toward compatibility with SQLAlchemy 2.0.
+
 - Reverted deprecation (:issue:`45324`) of behavior of :meth:`Series.__getitem__` and :meth:`Series.__setitem__` slicing with an integer :class:`Index`; this will remain positional (:issue:`49612`)
 -
 

doc/source/whatsnew/v2.0.0.rst

@@ -515,6 +515,7 @@ Other API changes
 - Default value of ``dtype`` in :func:`get_dummies` is changed to ``bool`` from ``uint8`` (:issue:`45848`)
 - :meth:`DataFrame.astype`, :meth:`Series.astype`, and :meth:`DatetimeIndex.astype` casting datetime64 data to any of "datetime64[s]", "datetime64[ms]", "datetime64[us]" will return an object with the given resolution instead of coercing back to "datetime64[ns]" (:issue:`48928`)
 - :meth:`DataFrame.astype`, :meth:`Series.astype`, and :meth:`DatetimeIndex.astype` casting timedelta64 data to any of "timedelta64[s]", "timedelta64[ms]", "timedelta64[us]" will return an object with the given resolution instead of coercing to "float64" dtype (:issue:`48963`)
+- :meth:`DatetimeIndex.astype`, :meth:`TimedeltaIndex.astype`, :meth:`PeriodIndex.astype` :meth:`Series.astype`, :meth:`DataFrame.astype` with ``datetime64``, ``timedelta64`` or :class:`PeriodDtype` dtypes no longer allow converting to integer dtypes other than "int64", do ``obj.astype('int64', copy=False).astype(dtype)`` instead (:issue:`49715`)
 - :meth:`Index.astype` now allows casting from ``float64`` dtype to datetime-like dtypes, matching :class:`Series` behavior (:issue:`49660`)
 - Passing data with dtype of "timedelta64[s]", "timedelta64[ms]", or "timedelta64[us]" to :class:`TimedeltaIndex`, :class:`Series`, or :class:`DataFrame` constructors will now retain that dtype instead of casting to "timedelta64[ns]"; timedelta64 data with lower resolution will be cast to the lowest supported resolution "timedelta64[s]" (:issue:`49014`)
 - Passing ``dtype`` of "timedelta64[s]", "timedelta64[ms]", or "timedelta64[us]" to :class:`TimedeltaIndex`, :class:`Series`, or :class:`DataFrame` constructors will now retain that dtype instead of casting to "timedelta64[ns]"; passing a dtype with lower resolution for :class:`Series` or :class:`DataFrame` will be cast to the lowest supported resolution "timedelta64[s]" (:issue:`49014`)

pandas/core/arrays/datetimelike.py

@@ -96,7 +96,6 @@
     is_period_dtype,
     is_string_dtype,
     is_timedelta64_dtype,
-    is_unsigned_integer_dtype,
     pandas_dtype,
 )
 from pandas.core.dtypes.dtypes import (
@@ -468,39 +467,10 @@ def astype(self, dtype, copy: bool = True):
             # we deliberately ignore int32 vs. int64 here.
             # See https://github.com/pandas-dev/pandas/issues/24381 for more.
             values = self.asi8
-
-            if is_unsigned_integer_dtype(dtype):
-                # Again, we ignore int32 vs. int64
-                values = values.view("uint64")
-                if dtype != np.uint64:
-                    # GH#45034
-                    warnings.warn(
-                        f"The behavior of .astype from {self.dtype} to {dtype} is "
-                        "deprecated. In a future version, this astype will return "
-                        "exactly the specified dtype instead of uint64, and will "
-                        "raise if that conversion overflows.",
-                        FutureWarning,
-                        stacklevel=find_stack_level(),
-                    )
-                elif (self.asi8 < 0).any():
-                    # GH#45034
-                    warnings.warn(
-                        f"The behavior of .astype from {self.dtype} to {dtype} is "
-                        "deprecated. In a future version, this astype will "
-                        "raise if the conversion overflows, as it did in this "
-                        "case with negative int64 values.",
-                        FutureWarning,
-                        stacklevel=find_stack_level(),
-                    )
-            elif dtype != np.int64:
-                # GH#45034
-                warnings.warn(
-                    f"The behavior of .astype from {self.dtype} to {dtype} is "
-                    "deprecated. In a future version, this astype will return "
-                    "exactly the specified dtype instead of int64, and will "
-                    "raise if that conversion overflows.",
-                    FutureWarning,
-                    stacklevel=find_stack_level(),
+            if dtype != np.int64:
+                raise TypeError(
+                    f"Converting from {self.dtype} to {dtype} is not supported. "
+                    "Do obj.astype('int64').astype(dtype) instead"
                 )
 
             if copy:

pandas/core/dtypes/astype.py

@@ -171,16 +171,6 @@ def astype_array(values: ArrayLike, dtype: DtypeObj, copy: bool = False) -> Arra
     -------
     ndarray or ExtensionArray
     """
-    if (
-        values.dtype.kind in ["m", "M"]
-        and dtype.kind in ["i", "u"]
-        and isinstance(dtype, np.dtype)
-        and dtype.itemsize != 8
-    ):
-        # TODO(2.0) remove special case once deprecation on DTA/TDA is enforced
-        msg = rf"cannot astype a datetimelike from [{values.dtype}] to [{dtype}]"
-        raise TypeError(msg)
-
     if is_dtype_equal(values.dtype, dtype):
         if copy:
             return values.copy()

pandas/core/dtypes/common.py

@@ -638,7 +638,11 @@ def is_any_int_dtype(arr_or_dtype) -> bool:
     >>> is_any_int_dtype(pd.Index([1, 2.]))  # float
     False
     """
-    return _is_dtype_type(arr_or_dtype, classes(np.integer, np.timedelta64))
+    return _is_dtype_type(
+        arr_or_dtype, classes(np.integer, np.timedelta64)
+    ) or _is_dtype(
+        arr_or_dtype, lambda typ: isinstance(typ, ExtensionDtype) and typ.kind in "iu"
+    )
 
 
 def is_integer_dtype(arr_or_dtype) -> bool:

pandas/tests/arrays/period/test_astype.py

@@ -14,26 +14,13 @@ def test_astype_int(dtype):
     # Period/Datetime/Timedelta astype
     arr = period_array(["2000", "2001", None], freq="D")
 
-    if np.dtype(dtype).kind == "u":
-        expected_dtype = np.dtype("uint64")
-        warn1 = FutureWarning
-    else:
-        expected_dtype = np.dtype("int64")
-        warn1 = None
-
-    msg_overflow = "will raise if the conversion overflows"
-    with tm.assert_produces_warning(warn1, match=msg_overflow):
-        expected = arr.astype(expected_dtype)
-
-    warn = None if dtype == expected_dtype else FutureWarning
-    msg = " will return exactly the specified dtype"
-    if warn is None and warn1 is not None:
-        warn = warn1
-        msg = msg_overflow
-    with tm.assert_produces_warning(warn, match=msg):
-        result = arr.astype(dtype)
-
-    assert result.dtype == expected_dtype
+    if np.dtype(dtype) != np.int64:
+        with pytest.raises(TypeError, match=r"Do obj.astype\('int64'\)"):
+            arr.astype(dtype)
+        return
+
+    result = arr.astype(dtype)
+    expected = arr._ndarray.view("i8")
     tm.assert_numpy_array_equal(result, expected)
 
 

pandas/tests/arrays/test_datetimes.py

@@ -377,20 +377,13 @@ def test_astype_copies(self, dtype, other):
     def test_astype_int(self, dtype):
         arr = DatetimeArray._from_sequence([pd.Timestamp("2000"), pd.Timestamp("2001")])
 
-        if np.dtype(dtype).kind == "u":
-            expected_dtype = np.dtype("uint64")
-        else:
-            expected_dtype = np.dtype("int64")
-        expected = arr.astype(expected_dtype)
-
-        warn = None
-        if dtype != expected_dtype:
-            warn = FutureWarning
-        msg = " will return exactly the specified dtype"
-        with tm.assert_produces_warning(warn, match=msg):
-            result = arr.astype(dtype)
+        if np.dtype(dtype) != np.int64:
+            with pytest.raises(TypeError, match=r"Do obj.astype\('int64'\)"):
+                arr.astype(dtype)
+            return
 
-        assert result.dtype == expected_dtype
+        result = arr.astype(dtype)
+        expected = arr._ndarray.view("i8")
         tm.assert_numpy_array_equal(result, expected)
 
     def test_astype_to_sparse_dt64(self):

pandas/tests/arrays/test_timedeltas.py

@@ -190,20 +190,13 @@ class TestTimedeltaArray:
     def test_astype_int(self, dtype):
         arr = TimedeltaArray._from_sequence([Timedelta("1H"), Timedelta("2H")])
 
-        if np.dtype(dtype).kind == "u":
-            expected_dtype = np.dtype("uint64")
-        else:
-            expected_dtype = np.dtype("int64")
-        expected = arr.astype(expected_dtype)
-
-        warn = None
-        if dtype != expected_dtype:
-            warn = FutureWarning
-        msg = " will return exactly the specified dtype"
-        with tm.assert_produces_warning(warn, match=msg):
-            result = arr.astype(dtype)
-
-        assert result.dtype == expected_dtype
+        if np.dtype(dtype) != np.int64:
+            with pytest.raises(TypeError, match=r"Do obj.astype\('int64'\)"):
+                arr.astype(dtype)
+            return
+
+        result = arr.astype(dtype)
+        expected = arr._ndarray.view("i8")
         tm.assert_numpy_array_equal(result, expected)
 
     def test_setitem_clears_freq(self):

pandas/tests/extension/test_arrow.py

@@ -35,6 +35,8 @@
 )
 from pandas.errors import PerformanceWarning
 
+from pandas.core.dtypes.common import is_any_int_dtype
+
 import pandas as pd
 import pandas._testing as tm
 from pandas.api.types import (
@@ -1459,6 +1461,15 @@ def test_is_integer_dtype(data):
         assert not is_integer_dtype(data)
 
 
+def test_is_any_integer_dtype(data):
+    # GH 50667
+    pa_type = data.dtype.pyarrow_dtype
+    if pa.types.is_integer(pa_type):
+        assert is_any_int_dtype(data)
+    else:
+        assert not is_any_int_dtype(data)
+
+
 def test_is_signed_integer_dtype(data):
     pa_type = data.dtype.pyarrow_dtype
     if pa.types.is_signed_integer(pa_type):

pandas/tests/frame/test_unary.py

@@ -3,6 +3,8 @@
 import numpy as np
 import pytest
 
+from pandas.compat import is_numpy_dev
+
 import pandas as pd
 import pandas._testing as tm
 
@@ -98,11 +100,6 @@ def test_pos_numeric(self, df):
     @pytest.mark.parametrize(
         "df",
         [
-            # numpy changing behavior in the future
-            pytest.param(
-                pd.DataFrame({"a": ["a", "b"]}),
-                marks=[pytest.mark.filterwarnings("ignore")],
-            ),
             pd.DataFrame({"a": np.array([-1, 2], dtype=object)}),
             pd.DataFrame({"a": [Decimal("-1.0"), Decimal("2.0")]}),
         ],
@@ -112,6 +109,25 @@ def test_pos_object(self, df):
         tm.assert_frame_equal(+df, df)
         tm.assert_series_equal(+df["a"], df["a"])
 
+    @pytest.mark.parametrize(
+        "df",
+        [
+            pytest.param(
+                pd.DataFrame({"a": ["a", "b"]}),
+                marks=[pytest.mark.filterwarnings("ignore")],
+            ),
+        ],
+    )
+    def test_pos_object_raises(self, df):
+        # GH#21380
+        if is_numpy_dev:
+            with pytest.raises(
+                TypeError, match=r"^bad operand type for unary \+: \'str\'$"
+            ):
+                tm.assert_frame_equal(+df, df)
+        else:
+            tm.assert_series_equal(+df["a"], df["a"])
+
     @pytest.mark.parametrize(
         "df", [pd.DataFrame({"a": pd.to_datetime(["2017-01-22", "1970-01-01"])})]
     )

pandas/tests/indexes/datetimes/methods/test_astype.py

@@ -15,10 +15,7 @@
     date_range,
 )
 import pandas._testing as tm
-from pandas.core.api import (
-    Int64Index,
-    UInt64Index,
-)
+from pandas.core.api import Int64Index
 
 
 class TestDatetimeIndex:
@@ -47,16 +44,11 @@ def test_astype(self):
 
     def test_astype_uint(self):
         arr = date_range("2000", periods=2, name="idx")
-        expected = UInt64Index(
-            np.array([946684800000000000, 946771200000000000], dtype="uint64"),
-            name="idx",
-        )
-        tm.assert_index_equal(arr.astype("uint64"), expected)
 
-        msg = "will return exactly the specified dtype instead of uint64"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            res = arr.astype("uint32")
-        tm.assert_index_equal(res, expected)
+        with pytest.raises(TypeError, match=r"Do obj.astype\('int64'\)"):
+            arr.astype("uint64")
+        with pytest.raises(TypeError, match=r"Do obj.astype\('int64'\)"):
+            arr.astype("uint32")
 
     def test_astype_with_tz(self):
 

pandas/tests/indexes/interval/test_astype.py

@@ -206,15 +206,18 @@ def index(self, request):
     def test_subtype_integer(self, index, subtype):
         dtype = IntervalDtype(subtype, "right")
 
-        warn = None
-        if index.isna().any() and subtype == "uint64":
-            warn = FutureWarning
-        msg = "In a future version, this astype will raise if the conversion overflows"
-
-        with tm.assert_produces_warning(warn, match=msg):
-            result = index.astype(dtype)
-            new_left = index.left.astype(subtype)
-            new_right = index.right.astype(subtype)
+        if subtype != "int64":
+            msg = (
+                r"Cannot convert interval\[(timedelta64|datetime64)\[ns.*\], .*\] "
+                r"to interval\[uint64, .*\]"
+            )
+            with pytest.raises(TypeError, match=msg):
+                index.astype(dtype)
+            return
+
+        result = index.astype(dtype)
+        new_left = index.left.astype(subtype)
+        new_right = index.right.astype(subtype)
 
         expected = IntervalIndex.from_arrays(new_left, new_right, closed=index.closed)
         tm.assert_index_equal(result, expected)

pandas/tests/indexes/period/methods/test_astype.py

@@ -11,10 +11,7 @@
     period_range,
 )
 import pandas._testing as tm
-from pandas.core.indexes.api import (
-    Int64Index,
-    UInt64Index,
-)
+from pandas.core.indexes.api import Int64Index
 
 
 class TestPeriodIndexAsType:
@@ -55,13 +52,11 @@ def test_astype_conversion(self):
 
     def test_astype_uint(self):
         arr = period_range("2000", periods=2, name="idx")
-        expected = UInt64Index(np.array([10957, 10958], dtype="uint64"), name="idx")
-        tm.assert_index_equal(arr.astype("uint64"), expected)
 
-        msg = "will return exactly the specified dtype instead of uint64"
-        with tm.assert_produces_warning(FutureWarning, match=msg):
-            res = arr.astype("uint32")
-        tm.assert_index_equal(res, expected)
+        with pytest.raises(TypeError, match=r"Do obj.astype\('int64'\)"):
+            arr.astype("uint64")
+        with pytest.raises(TypeError, match=r"Do obj.astype\('int64'\)"):
+            arr.astype("uint32")
 
     def test_astype_object(self):
         idx = PeriodIndex([], freq="M")