Merge branch 'main' into cow_indexing_midx_slice

Author: phofl

doc/source/whatsnew/v2.0.0.rst

@@ -209,6 +209,8 @@ Copy-on-Write improvements
 - :meth:`DataFrame.__getitem__` will now respect the Copy-on-Write mechanism when the
   :class:`DataFrame` has :class:`MultiIndex` columns.
 
+- :meth:`Series.view` will now respect the Copy-on-Write mechanism.
+
 Copy-on-Write can be enabled through one of
 
 .. code-block:: python

doc/source/whatsnew/v2.1.0.rst

@@ -201,6 +201,7 @@ Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
 - Bug in :meth:`DataFrameGroupBy.idxmin`, :meth:`SeriesGroupBy.idxmin`, :meth:`DataFrameGroupBy.idxmax`, :meth:`SeriesGroupBy.idxmax` return wrong dtype when used on empty DataFrameGroupBy or SeriesGroupBy (:issue:`51423`)
 - Bug in weighted rolling aggregations when specifying ``min_periods=0`` (:issue:`51449`)
+- Bug in :meth:`DataFrame.resample` and :meth:`Series.resample` in incorrectly allowing non-fixed ``freq`` when resampling on a :class:`TimedeltaIndex` (:issue:`51896`)
 -
 
 Reshaping
@@ -227,6 +228,7 @@ Styler
 
 Other
 ^^^^^
+-
 
 .. ***DO NOT USE THIS SECTION***
 

environment.yml

@@ -118,3 +118,4 @@ dependencies:
 
   - pip:
       - sphinx-toggleprompt
+      - typing_extensions; python_version<"3.11"

pandas/_libs/sparse.pyi

@@ -1,13 +1,11 @@
-from typing import (
-    Sequence,
-    TypeVar,
-)
+from typing import Sequence
 
 import numpy as np
 
-from pandas._typing import npt
-
-_SparseIndexT = TypeVar("_SparseIndexT", bound=SparseIndex)
+from pandas._typing import (
+    Self,
+    npt,
+)
 
 class SparseIndex:
     length: int
@@ -24,8 +22,8 @@ class SparseIndex:
     def lookup_array(self, indexer: npt.NDArray[np.int32]) -> npt.NDArray[np.int32]: ...
     def to_int_index(self) -> IntIndex: ...
     def to_block_index(self) -> BlockIndex: ...
-    def intersect(self: _SparseIndexT, y_: SparseIndex) -> _SparseIndexT: ...
-    def make_union(self: _SparseIndexT, y_: SparseIndex) -> _SparseIndexT: ...
+    def intersect(self, y_: SparseIndex) -> Self: ...
+    def make_union(self, y_: SparseIndex) -> Self: ...
 
 class IntIndex(SparseIndex):
     indices: npt.NDArray[np.int32]

pandas/_libs/tslibs/offsets.pyi

@@ -13,7 +13,10 @@ from typing import (
 import numpy as np
 
 from pandas._libs.tslibs.nattype import NaTType
-from pandas._typing import npt
+from pandas._typing import (
+    Self,
+    npt,
+)
 
 from .timedeltas import Timedelta
 
@@ -39,40 +42,40 @@ class BaseOffset:
     @overload
     def __add__(self, other: npt.NDArray[np.object_]) -> npt.NDArray[np.object_]: ...
     @overload
-    def __add__(self: _BaseOffsetT, other: BaseOffset) -> _BaseOffsetT: ...
+    def __add__(self, other: BaseOffset) -> Self: ...
     @overload
     def __add__(self, other: _DatetimeT) -> _DatetimeT: ...
     @overload
     def __add__(self, other: _TimedeltaT) -> _TimedeltaT: ...
     @overload
     def __radd__(self, other: npt.NDArray[np.object_]) -> npt.NDArray[np.object_]: ...
     @overload
-    def __radd__(self: _BaseOffsetT, other: BaseOffset) -> _BaseOffsetT: ...
+    def __radd__(self, other: BaseOffset) -> Self: ...
     @overload
     def __radd__(self, other: _DatetimeT) -> _DatetimeT: ...
     @overload
     def __radd__(self, other: _TimedeltaT) -> _TimedeltaT: ...
     @overload
     def __radd__(self, other: NaTType) -> NaTType: ...
-    def __sub__(self: _BaseOffsetT, other: BaseOffset) -> _BaseOffsetT: ...
+    def __sub__(self, other: BaseOffset) -> Self: ...
     @overload
     def __rsub__(self, other: npt.NDArray[np.object_]) -> npt.NDArray[np.object_]: ...
     @overload
-    def __rsub__(self: _BaseOffsetT, other: BaseOffset) -> _BaseOffsetT: ...
+    def __rsub__(self, other: BaseOffset): ...
     @overload
     def __rsub__(self, other: _DatetimeT) -> _DatetimeT: ...
     @overload
     def __rsub__(self, other: _TimedeltaT) -> _TimedeltaT: ...
     @overload
     def __mul__(self, other: np.ndarray) -> np.ndarray: ...
     @overload
-    def __mul__(self: _BaseOffsetT, other: int) -> _BaseOffsetT: ...
+    def __mul__(self, other: int): ...
     @overload
     def __rmul__(self, other: np.ndarray) -> np.ndarray: ...
     @overload
-    def __rmul__(self: _BaseOffsetT, other: int) -> _BaseOffsetT: ...
-    def __neg__(self: _BaseOffsetT) -> _BaseOffsetT: ...
-    def copy(self: _BaseOffsetT) -> _BaseOffsetT: ...
+    def __rmul__(self, other: int) -> Self: ...
+    def __neg__(self) -> Self: ...
+    def copy(self) -> Self: ...
     @property
     def name(self) -> str: ...
     @property

pandas/_libs/tslibs/timedeltas.pyi

@@ -12,7 +12,10 @@ from pandas._libs.tslibs import (
     NaTType,
     Tick,
 )
-from pandas._typing import npt
+from pandas._typing import (
+    Self,
+    npt,
+)
 
 # This should be kept consistent with the keys in the dict timedelta_abbrevs
 # in pandas/_libs/tslibs/timedeltas.pyx
@@ -111,9 +114,9 @@ class Timedelta(timedelta):
     @property
     def asm8(self) -> np.timedelta64: ...
     # TODO: round/floor/ceil could return NaT?
-    def round(self: _S, freq: str) -> _S: ...
-    def floor(self: _S, freq: str) -> _S: ...
-    def ceil(self: _S, freq: str) -> _S: ...
+    def round(self, freq: str) -> Self: ...
+    def floor(self, freq: str) -> Self: ...
+    def ceil(self, freq: str) -> Self: ...
     @property
     def resolution_string(self) -> str: ...
     def __add__(self, other: timedelta) -> Timedelta: ...

pandas/_libs/tslibs/timestamps.pyi

@@ -21,6 +21,7 @@ from pandas._libs.tslibs import (
     Tick,
     Timedelta,
 )
+from pandas._typing import Self
 
 _DatetimeT = TypeVar("_DatetimeT", bound=datetime)
 
@@ -80,30 +81,28 @@ class Timestamp(datetime):
     @property
     def fold(self) -> int: ...
     @classmethod
-    def fromtimestamp(
-        cls: type[_DatetimeT], ts: float, tz: _tzinfo | None = ...
-    ) -> _DatetimeT: ...
+    def fromtimestamp(cls, ts: float, tz: _tzinfo | None = ...) -> Self: ...
     @classmethod
-    def utcfromtimestamp(cls: type[_DatetimeT], ts: float) -> _DatetimeT: ...
+    def utcfromtimestamp(cls, ts: float) -> Self: ...
     @classmethod
-    def today(cls: type[_DatetimeT], tz: _tzinfo | str | None = ...) -> _DatetimeT: ...
+    def today(cls, tz: _tzinfo | str | None = ...) -> Self: ...
     @classmethod
     def fromordinal(
-        cls: type[_DatetimeT],
+        cls,
         ordinal: int,
         tz: _tzinfo | str | None = ...,
-    ) -> _DatetimeT: ...
+    ) -> Self: ...
     @classmethod
-    def now(cls: type[_DatetimeT], tz: _tzinfo | str | None = ...) -> _DatetimeT: ...
+    def now(cls, tz: _tzinfo | str | None = ...) -> Self: ...
     @classmethod
-    def utcnow(cls: type[_DatetimeT]) -> _DatetimeT: ...
+    def utcnow(cls) -> Self: ...
     # error: Signature of "combine" incompatible with supertype "datetime"
     @classmethod
     def combine(  # type: ignore[override]
         cls, date: _date, time: _time
     ) -> datetime: ...
     @classmethod
-    def fromisoformat(cls: type[_DatetimeT], date_string: str) -> _DatetimeT: ...
+    def fromisoformat(cls, date_string: str) -> Self: ...
     def strftime(self, format: str) -> str: ...
     def __format__(self, fmt: str) -> str: ...
     def toordinal(self) -> int: ...
@@ -116,7 +115,7 @@ class Timestamp(datetime):
     # LSP violation: nanosecond is not present in datetime.datetime.replace
     # and has positional args following it
     def replace(  # type: ignore[override]
-        self: _DatetimeT,
+        self,
         year: int | None = ...,
         month: int | None = ...,
         day: int | None = ...,
@@ -127,11 +126,9 @@ class Timestamp(datetime):
         nanosecond: int | None = ...,
         tzinfo: _tzinfo | type[object] | None = ...,
         fold: int | None = ...,
-    ) -> _DatetimeT: ...
+    ) -> Self: ...
     # LSP violation: datetime.datetime.astimezone has a default value for tz
-    def astimezone(  # type: ignore[override]
-        self: _DatetimeT, tz: _tzinfo | None
-    ) -> _DatetimeT: ...
+    def astimezone(self, tz: _tzinfo | None) -> Self: ...  # type: ignore[override]
     def ctime(self) -> str: ...
     def isoformat(self, sep: str = ..., timespec: str = ...) -> str: ...
     @classmethod
@@ -147,16 +144,12 @@ class Timestamp(datetime):
     @overload  # type: ignore[override]
     def __add__(self, other: np.ndarray) -> np.ndarray: ...
     @overload
-    def __add__(
-        self: _DatetimeT, other: timedelta | np.timedelta64 | Tick
-    ) -> _DatetimeT: ...
-    def __radd__(self: _DatetimeT, other: timedelta) -> _DatetimeT: ...
+    def __add__(self, other: timedelta | np.timedelta64 | Tick) -> Self: ...
+    def __radd__(self, other: timedelta) -> Self: ...
     @overload  # type: ignore[override]
     def __sub__(self, other: datetime) -> Timedelta: ...
     @overload
-    def __sub__(
-        self: _DatetimeT, other: timedelta | np.timedelta64 | Tick
-    ) -> _DatetimeT: ...
+    def __sub__(self, other: timedelta | np.timedelta64 | Tick) -> Self: ...
     def __hash__(self) -> int: ...
     def weekday(self) -> int: ...
     def isoweekday(self) -> int: ...
@@ -181,25 +174,25 @@ class Timestamp(datetime):
     def to_julian_date(self) -> np.float64: ...
     @property
     def asm8(self) -> np.datetime64: ...
-    def tz_convert(self: _DatetimeT, tz: _tzinfo | str | None) -> _DatetimeT: ...
+    def tz_convert(self, tz: _tzinfo | str | None) -> Self: ...
     # TODO: could return NaT?
     def tz_localize(
-        self: _DatetimeT,
+        self,
         tz: _tzinfo | str | None,
         ambiguous: str = ...,
         nonexistent: str = ...,
-    ) -> _DatetimeT: ...
-    def normalize(self: _DatetimeT) -> _DatetimeT: ...
+    ) -> Self: ...
+    def normalize(self) -> Self: ...
     # TODO: round/floor/ceil could return NaT?
     def round(
-        self: _DatetimeT, freq: str, ambiguous: bool | str = ..., nonexistent: str = ...
-    ) -> _DatetimeT: ...
+        self, freq: str, ambiguous: bool | str = ..., nonexistent: str = ...
+    ) -> Self: ...
     def floor(
-        self: _DatetimeT, freq: str, ambiguous: bool | str = ..., nonexistent: str = ...
-    ) -> _DatetimeT: ...
+        self, freq: str, ambiguous: bool | str = ..., nonexistent: str = ...
+    ) -> Self: ...
     def ceil(
-        self: _DatetimeT, freq: str, ambiguous: bool | str = ..., nonexistent: str = ...
-    ) -> _DatetimeT: ...
+        self, freq: str, ambiguous: bool | str = ..., nonexistent: str = ...
+    ) -> Self: ...
     def day_name(self, locale: str | None = ...) -> str: ...
     def month_name(self, locale: str | None = ...) -> str: ...
     @property

pandas/_typing.py

@@ -6,6 +6,7 @@
     tzinfo,
 )
 from os import PathLike
+import sys
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -83,8 +84,13 @@
     # Name "npt._ArrayLikeInt_co" is not defined  [name-defined]
     NumpySorter = Optional[npt._ArrayLikeInt_co]  # type: ignore[name-defined]
 
+    if sys.version_info >= (3, 11):
+        from typing import Self
+    else:
+        from typing_extensions import Self  # pyright: reportUnusedImport = false
 else:
     npt: Any = None
+    Self: Any = None
 
 HashableT = TypeVar("HashableT", bound=Hashable)
 

pandas/core/arrays/datetimelike.py

@@ -1851,6 +1851,8 @@ def __init__(
             values = values.copy()
         if freq:
             freq = to_offset(freq)
+            if values.dtype.kind == "m" and not isinstance(freq, Tick):
+                raise TypeError("TimedeltaArray/Index freq must be a Tick")
 
         NDArrayBacked.__init__(self, values=values, dtype=dtype)
         self._freq = freq
@@ -1874,6 +1876,8 @@ def freq(self, value) -> None:
         if value is not None:
             value = to_offset(value)
             self._validate_frequency(self, value)
+            if self.dtype.kind == "m" and not isinstance(value, Tick):
+                raise TypeError("TimedeltaArray/Index freq must be a Tick")
 
             if self.ndim > 1:
                 raise ValueError("Cannot set freq with ndim > 1")
@@ -2067,9 +2071,9 @@ def _with_freq(self, freq):
             # Always valid
             pass
         elif len(self) == 0 and isinstance(freq, BaseOffset):
-            # Always valid.  In the TimedeltaArray case, we assume this
-            #  is a Tick offset.
-            pass
+            # Always valid.  In the TimedeltaArray case, we require a Tick offset
+            if self.dtype.kind == "m" and not isinstance(freq, Tick):
+                raise TypeError("TimedeltaArray/Index freq must be a Tick")
         else:
             # As an internal method, we can ensure this assertion always holds
             assert freq == "infer"

pandas/core/arrays/timedeltas.py

@@ -202,6 +202,7 @@ def _simple_new(  # type: ignore[override]
         assert not tslibs.is_unitless(dtype)
         assert isinstance(values, np.ndarray), type(values)
         assert dtype == values.dtype
+        assert freq is None or isinstance(freq, Tick)
 
         result = super()._simple_new(values=values, dtype=dtype)
         result._freq = freq

pandas/core/resample.py

@@ -1826,6 +1826,13 @@ def _get_time_delta_bins(self, ax: TimedeltaIndex):
                 f"an instance of {type(ax).__name__}"
             )
 
+        if not isinstance(self.freq, Tick):
+            # GH#51896
+            raise ValueError(
+                "Resampling on a TimedeltaIndex requires fixed-duration `freq`, "
+                f"e.g. '24H' or '3D', not {self.freq}"
+            )
+
         if not len(ax):
             binner = labels = TimedeltaIndex(data=[], freq=self.freq, name=ax.name)
             return binner, [], labels

pandas/core/series.py

@@ -839,6 +839,10 @@ def view(self, dtype: Dtype | None = None) -> Series:
         #  implementation
         res_values = self.array.view(dtype)
         res_ser = self._constructor(res_values, index=self.index)
+        if isinstance(res_ser._mgr, SingleBlockManager) and using_copy_on_write():
+            blk = res_ser._mgr._block
+            blk.refs = cast("BlockValuesRefs", self._references)
+            blk.refs.add_reference(blk)  # type: ignore[arg-type]
         return res_ser.__finalize__(self, method="view")
 
     # ----------------------------------------------------------------------

pandas/tests/copy_view/test_methods.py

@@ -1672,3 +1672,21 @@ def test_transpose_ea_single_column(using_copy_on_write):
     result = df.T
 
     assert not np.shares_memory(get_array(df, "a"), get_array(result, 0))
+
+
+def test_series_view(using_copy_on_write):
+    ser = Series([1, 2, 3])
+    ser_orig = ser.copy()
+
+    ser2 = ser.view()
+    assert np.shares_memory(get_array(ser), get_array(ser2))
+    if using_copy_on_write:
+        assert not ser2._mgr._has_no_reference(0)
+
+    ser2.iloc[0] = 100
+
+    if using_copy_on_write:
+        tm.assert_series_equal(ser_orig, ser)
+    else:
+        expected = Series([100, 2, 3])
+        tm.assert_series_equal(ser, expected)