DEPR: astype dt64<->dt64tz (#39258)

Author: jbrockmendel

doc/source/user_guide/timeseries.rst

@@ -2605,17 +2605,10 @@ For example, to localize and convert a naive stamp to time zone aware.
    s_naive.dt.tz_localize("UTC").dt.tz_convert("US/Eastern")
 
 Time zone information can also be manipulated using the ``astype`` method.
-This method can localize and convert time zone naive timestamps or
-convert time zone aware timestamps.
+This method can convert between different timezone-aware dtypes.
 
 .. ipython:: python
 
-   # localize and convert a naive time zone
-   s_naive.astype("datetime64[ns, US/Eastern]")
-
-   # make an aware tz naive
-   s_aware.astype("datetime64[ns]")
-
    # convert to a new time zone
    s_aware.astype("datetime64[ns, CET]")
 

doc/source/whatsnew/v1.3.0.rst

@@ -193,6 +193,7 @@ Deprecations
 - Deprecated :attr:`Rolling.win_type` returning ``"freq"`` (:issue:`38963`)
 - Deprecated :attr:`Rolling.is_datetimelike` (:issue:`38963`)
 - Deprecated :meth:`core.window.ewm.ExponentialMovingWindow.vol` (:issue:`39220`)
+- Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :class:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`)
 -
 
 .. ---------------------------------------------------------------------------

pandas/core/dtypes/cast.py

@@ -38,6 +38,7 @@
     tz_compare,
 )
 from pandas._typing import AnyArrayLike, ArrayLike, Dtype, DtypeObj, Scalar
+from pandas.util._exceptions import find_stack_level
 from pandas.util._validators import validate_bool_kwarg
 
 from pandas.core.dtypes.common import (
@@ -964,6 +965,16 @@ def astype_dt64_to_dt64tz(
         if copy:
             # this should be the only copy
             values = values.copy()
+
+        level = find_stack_level()
+        warnings.warn(
+            "Using .astype to convert from timezone-naive dtype to "
+            "timezone-aware dtype is deprecated and will raise in a "
+            "future version.  Use ser.dt.tz_localize instead.",
+            FutureWarning,
+            stacklevel=level,
+        )
+
         # FIXME: GH#33401 this doesn't match DatetimeArray.astype, which
         #  goes through the `not via_utc` path
         return values.tz_localize("UTC").tz_convert(dtype.tz)
@@ -973,6 +984,15 @@ def astype_dt64_to_dt64tz(
 
         if values.tz is None and aware:
             dtype = cast(DatetimeTZDtype, dtype)
+            level = find_stack_level()
+            warnings.warn(
+                "Using .astype to convert from timezone-naive dtype to "
+                "timezone-aware dtype is deprecated and will raise in a "
+                "future version.  Use obj.tz_localize instead.",
+                FutureWarning,
+                stacklevel=level,
+            )
+
             return values.tz_localize(dtype.tz)
 
         elif aware:
@@ -984,6 +1004,16 @@ def astype_dt64_to_dt64tz(
             return result
 
         elif values.tz is not None:
+            level = find_stack_level()
+            warnings.warn(
+                "Using .astype to convert from timezone-aware dtype to "
+                "timezone-naive dtype is deprecated and will raise in a "
+                "future version.  Use obj.tz_localize(None) or "
+                "obj.tz_convert('UTC').tz_localize(None) instead",
+                FutureWarning,
+                stacklevel=level,
+            )
+
             result = values.tz_convert("UTC").tz_localize(None)
             if copy:
                 result = result.copy()

pandas/core/dtypes/dtypes.py

@@ -440,7 +440,7 @@ def _hash_categories(categories, ordered: Ordered = True) -> int:
 
             if DatetimeTZDtype.is_dtype(categories.dtype):
                 # Avoid future warning.
-                categories = categories.astype("datetime64[ns]")
+                categories = categories.view("datetime64[ns]")
 
             cat_array = hash_array(np.asarray(categories), categorize=False)
         if ordered:

pandas/core/internals/blocks.py

@@ -2175,6 +2175,9 @@ def get_values(self, dtype: Optional[Dtype] = None):
     def external_values(self):
         # NB: this is different from np.asarray(self.values), since that
         #  return an object-dtype ndarray of Timestamps.
+        if self.is_datetimetz:
+            # avoid FutureWarning in .astype in casting from dt64t to dt64
+            return self.values._data
         return np.asarray(self.values.astype("datetime64[ns]", copy=False))
 
     def fillna(self, value, limit=None, inplace=False, downcast=None):

pandas/tests/arrays/test_datetimes.py

@@ -175,11 +175,18 @@ def test_astype_to_same(self):
     )
     def test_astype_copies(self, dtype, other):
         # https://github.com/pandas-dev/pandas/pull/32490
-        s = pd.Series([1, 2], dtype=dtype)
-        orig = s.copy()
-        t = s.astype(other)
+        ser = pd.Series([1, 2], dtype=dtype)
+        orig = ser.copy()
+
+        warn = None
+        if (dtype == "datetime64[ns]") ^ (other == "datetime64[ns]"):
+            # deprecated in favor of tz_localize
+            warn = FutureWarning
+
+        with tm.assert_produces_warning(warn):
+            t = ser.astype(other)
         t[:] = pd.NaT
-        tm.assert_series_equal(s, orig)
+        tm.assert_series_equal(ser, orig)
 
     @pytest.mark.parametrize("dtype", [int, np.int32, np.int64, "uint32", "uint64"])
     def test_astype_int(self, dtype):

pandas/tests/frame/methods/test_astype.py

@@ -515,7 +515,9 @@ def test_astype_dt64tz(self, timezone_frame):
         result = timezone_frame.astype(object)
         tm.assert_frame_equal(result, expected)
 
-        result = timezone_frame.astype("datetime64[ns]")
+        with tm.assert_produces_warning(FutureWarning):
+            # dt64tz->dt64 deprecated
+            result = timezone_frame.astype("datetime64[ns]")
         expected = DataFrame(
             {
                 "A": date_range("20130101", periods=3),

pandas/tests/indexes/datetimes/methods/test_astype.py

@@ -58,7 +58,13 @@ def test_astype_with_tz(self):
 
         # with tz
         rng = date_range("1/1/2000", periods=10, tz="US/Eastern")
-        result = rng.astype("datetime64[ns]")
+        with tm.assert_produces_warning(FutureWarning):
+            # deprecated
+            result = rng.astype("datetime64[ns]")
+        with tm.assert_produces_warning(FutureWarning):
+            # check DatetimeArray while we're here deprecated
+            rng._data.astype("datetime64[ns]")
+
         expected = (
             date_range("1/1/2000", periods=10, tz="US/Eastern")
             .tz_convert("UTC")
@@ -78,7 +84,13 @@ def test_astype_tznaive_to_tzaware(self):
         # GH 18951: tz-naive to tz-aware
         idx = date_range("20170101", periods=4)
         idx = idx._with_freq(None)  # tz_localize does not preserve freq
-        result = idx.astype("datetime64[ns, US/Eastern]")
+        with tm.assert_produces_warning(FutureWarning):
+            # dt64->dt64tz deprecated
+            result = idx.astype("datetime64[ns, US/Eastern]")
+        with tm.assert_produces_warning(FutureWarning):
+            # dt64->dt64tz deprecated
+            idx._data.astype("datetime64[ns, US/Eastern]")
+
         expected = date_range("20170101", periods=4, tz="US/Eastern")
         expected = expected._with_freq(None)
         tm.assert_index_equal(result, expected)
@@ -155,7 +167,9 @@ def test_astype_datetime64(self):
         assert result is idx
 
         idx_tz = DatetimeIndex(["2016-05-16", "NaT", NaT, np.NaN], tz="EST", name="idx")
-        result = idx_tz.astype("datetime64[ns]")
+        with tm.assert_produces_warning(FutureWarning):
+            # dt64tz->dt64 deprecated
+            result = idx_tz.astype("datetime64[ns]")
         expected = DatetimeIndex(
             ["2016-05-16 05:00:00", "NaT", "NaT", "NaT"],
             dtype="datetime64[ns]",

pandas/tests/indexes/test_base.py

@@ -349,14 +349,18 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, klass):
         index = index.tz_localize(tz_naive_fixture)
         dtype = index.dtype
 
+        warn = None if tz_naive_fixture is None else FutureWarning
+        # astype dt64 -> dt64tz deprecated
+
         if attr == "asi8":
             result = DatetimeIndex(arg).tz_localize(tz_naive_fixture)
         else:
             result = klass(arg, tz=tz_naive_fixture)
         tm.assert_index_equal(result, index)
 
         if attr == "asi8":
-            result = DatetimeIndex(arg).astype(dtype)
+            with tm.assert_produces_warning(warn):
+                result = DatetimeIndex(arg).astype(dtype)
         else:
             result = klass(arg, dtype=dtype)
         tm.assert_index_equal(result, index)
@@ -368,7 +372,8 @@ def test_constructor_dtypes_datetime(self, tz_naive_fixture, attr, klass):
         tm.assert_index_equal(result, index)
 
         if attr == "asi8":
-            result = DatetimeIndex(list(arg)).astype(dtype)
+            with tm.assert_produces_warning(warn):
+                result = DatetimeIndex(list(arg)).astype(dtype)
         else:
             result = klass(list(arg), dtype=dtype)
         tm.assert_index_equal(result, index)

pandas/tests/indexes/test_common.py

@@ -352,6 +352,13 @@ def test_astype_preserves_name(self, index, dtype):
         if dtype in ["int64", "uint64"]:
             if needs_i8_conversion(index.dtype):
                 warn = FutureWarning
+        elif (
+            isinstance(index, DatetimeIndex)
+            and index.tz is not None
+            and dtype == "datetime64[ns]"
+        ):
+            # This astype is deprecated in favor of tz_localize
+            warn = FutureWarning
         try:
             # Some of these conversions cannot succeed so we use a try / except
             with tm.assert_produces_warning(warn, check_stacklevel=False):

pandas/tests/series/methods/test_astype.py

@@ -193,10 +193,14 @@ def test_astype_datetime64tz(self):
         tm.assert_series_equal(result, expected)
 
         # astype - datetime64[ns, tz]
-        result = Series(s.values).astype("datetime64[ns, US/Eastern]")
+        with tm.assert_produces_warning(FutureWarning):
+            # dt64->dt64tz astype deprecated
+            result = Series(s.values).astype("datetime64[ns, US/Eastern]")
         tm.assert_series_equal(result, s)
 
-        result = Series(s.values).astype(s.dtype)
+        with tm.assert_produces_warning(FutureWarning):
+            # dt64->dt64tz astype deprecated
+            result = Series(s.values).astype(s.dtype)
         tm.assert_series_equal(result, s)
 
         result = s.astype("datetime64[ns, CET]")

pandas/tests/series/methods/test_convert_dtypes.py

@@ -156,8 +156,18 @@ class TestSeriesConvertDtypes:
     def test_convert_dtypes(
         self, data, maindtype, params, expected_default, expected_other
     ):
+        warn = None
+        if (
+            hasattr(data, "dtype")
+            and data.dtype == "M8[ns]"
+            and isinstance(maindtype, pd.DatetimeTZDtype)
+        ):
+            # this astype is deprecated in favor of tz_localize
+            warn = FutureWarning
+
         if maindtype is not None:
-            series = pd.Series(data, dtype=maindtype)
+            with tm.assert_produces_warning(warn):
+                series = pd.Series(data, dtype=maindtype)
         else:
             series = pd.Series(data)
 
@@ -177,7 +187,17 @@ def test_convert_dtypes(
             if all(params_dict[key] is val for key, val in zip(spec[::2], spec[1::2])):
                 expected_dtype = dtype
 
-        expected = pd.Series(data, dtype=expected_dtype)
+        warn2 = None
+        if (
+            hasattr(data, "dtype")
+            and data.dtype == "M8[ns]"
+            and isinstance(expected_dtype, pd.DatetimeTZDtype)
+        ):
+            # this astype is deprecated in favor of tz_localize
+            warn2 = FutureWarning
+
+        with tm.assert_produces_warning(warn2):
+            expected = pd.Series(data, dtype=expected_dtype)
         tm.assert_series_equal(result, expected)
 
         # Test that it is a copy

pandas/util/_exceptions.py

@@ -1,4 +1,5 @@
 import contextlib
+import inspect
 from typing import Tuple
 
 
@@ -17,3 +18,27 @@ def rewrite_exception(old_name: str, new_name: str):
             args = args + err.args[1:]
         err.args = args
         raise
+
+
+def find_stack_level() -> int:
+    """
+    Find the appropriate stacklevel with which to issue a warning for astype.
+    """
+    stack = inspect.stack()
+
+    # find the lowest-level "astype" call that got us here
+    for n in range(2, 6):
+        if stack[n].function == "astype":
+            break
+
+    while stack[n].function in ["astype", "apply", "_astype"]:
+        # e.g.
+        #  bump up Block.astype -> BlockManager.astype -> NDFrame.astype
+        #  bump up Datetime.Array.astype -> DatetimeIndex.astype
+        n += 1
+
+    if stack[n].function == "__init__":
+        # Series.__init__
+        n += 1
+
+    return n