Backport PR #39069: REGR: diff_2d raising for int8, int16

Author: mzeitlin11

doc/source/whatsnew/v1.2.1.rst

@@ -24,6 +24,7 @@ Fixed regressions
 - Fixed regression in :meth:`.GroupBy.sem` where the presence of non-numeric columns would cause an error instead of being dropped (:issue:`38774`)
 - Fixed regression in :func:`read_excel` with non-rawbyte file handles (:issue:`38788`)
 - Bug in :meth:`read_csv` with ``float_precision="high"`` caused segfault or wrong parsing of long exponent strings. This resulted in a regression in some cases as the default for ``float_precision`` was changed in pandas 1.2.0 (:issue:`38753`)
+- Fixed regression in :meth:`DataFrameGroupBy.diff` raising for ``int8`` and ``int16`` columns (:issue:`39050`)
 - Fixed regression in :meth:`Rolling.skew` and :meth:`Rolling.kurt` modifying the object inplace (:issue:`38908`)
 - Fixed regression in :meth:`read_csv` and other read functions were the encoding error policy (``errors``) did not default to ``"replace"`` when no encoding was specified (:issue:`38989`)
 - Fixed regression in :meth:`DataFrame.replace` raising ValueError when :class:`DataFrame` has dtype ``bytes`` (:issue:`38900`)

pandas/core/algorithms.py

@@ -1981,7 +1981,13 @@ def diff(arr, n: int, axis: int = 0, stacklevel=3):
 
     elif is_integer_dtype(dtype):
         # We have to cast in order to be able to hold np.nan
-        dtype = np.float64
+
+        # int8, int16 are incompatible with float64,
+        # see https://github.com/cython/cython/issues/2646
+        if arr.dtype.name in ["int8", "int16"]:
+            dtype = np.float32
+        else:
+            dtype = np.float64
 
     orig_ndim = arr.ndim
     if orig_ndim == 1:

pandas/tests/groupby/test_groupby.py

@@ -1697,64 +1697,6 @@ def test_sort(x):
     g.apply(test_sort)
 
 
-def test_group_shift_with_null_key():
-    # This test is designed to replicate the segfault in issue #13813.
-    n_rows = 1200
-
-    # Generate a moderately large dataframe with occasional missing
-    # values in column `B`, and then group by [`A`, `B`]. This should
-    # force `-1` in `labels` array of `g.grouper.group_info` exactly
-    # at those places, where the group-by key is partially missing.
-    df = DataFrame(
-        [(i % 12, i % 3 if i % 3 else np.nan, i) for i in range(n_rows)],
-        dtype=float,
-        columns=["A", "B", "Z"],
-        index=None,
-    )
-    g = df.groupby(["A", "B"])
-
-    expected = DataFrame(
-        [(i + 12 if i % 3 and i < n_rows - 12 else np.nan) for i in range(n_rows)],
-        dtype=float,
-        columns=["Z"],
-        index=None,
-    )
-    result = g.shift(-1)
-
-    tm.assert_frame_equal(result, expected)
-
-
-def test_group_shift_with_fill_value():
-    # GH #24128
-    n_rows = 24
-    df = DataFrame(
-        [(i % 12, i % 3, i) for i in range(n_rows)],
-        dtype=float,
-        columns=["A", "B", "Z"],
-        index=None,
-    )
-    g = df.groupby(["A", "B"])
-
-    expected = DataFrame(
-        [(i + 12 if i < n_rows - 12 else 0) for i in range(n_rows)],
-        dtype=float,
-        columns=["Z"],
-        index=None,
-    )
-    result = g.shift(-1, fill_value=0)[["Z"]]
-
-    tm.assert_frame_equal(result, expected)
-
-
-def test_group_shift_lose_timezone():
-    # GH 30134
-    now_dt = Timestamp.utcnow()
-    df = DataFrame({"a": [1, 1], "date": now_dt})
-    result = df.groupby("a").shift(0).iloc[0]
-    expected = Series({"date": now_dt}, name=result.name)
-    tm.assert_series_equal(result, expected)
-
-
 def test_pivot_table_values_key_error():
     # This test is designed to replicate the error in issue #14938
     df = DataFrame(

pandas/tests/groupby/test_groupby_shift_diff.py

@@ -0,0 +1,106 @@
+import numpy as np
+import pytest
+
+from pandas import DataFrame, NaT, Series, Timedelta, Timestamp
+import pandas._testing as tm
+
+
+def test_group_shift_with_null_key():
+    # This test is designed to replicate the segfault in issue #13813.
+    n_rows = 1200
+
+    # Generate a moderately large dataframe with occasional missing
+    # values in column `B`, and then group by [`A`, `B`]. This should
+    # force `-1` in `labels` array of `g.grouper.group_info` exactly
+    # at those places, where the group-by key is partially missing.
+    df = DataFrame(
+        [(i % 12, i % 3 if i % 3 else np.nan, i) for i in range(n_rows)],
+        dtype=float,
+        columns=["A", "B", "Z"],
+        index=None,
+    )
+    g = df.groupby(["A", "B"])
+
+    expected = DataFrame(
+        [(i + 12 if i % 3 and i < n_rows - 12 else np.nan) for i in range(n_rows)],
+        dtype=float,
+        columns=["Z"],
+        index=None,
+    )
+    result = g.shift(-1)
+
+    tm.assert_frame_equal(result, expected)
+
+
+def test_group_shift_with_fill_value():
+    # GH #24128
+    n_rows = 24
+    df = DataFrame(
+        [(i % 12, i % 3, i) for i in range(n_rows)],
+        dtype=float,
+        columns=["A", "B", "Z"],
+        index=None,
+    )
+    g = df.groupby(["A", "B"])
+
+    expected = DataFrame(
+        [(i + 12 if i < n_rows - 12 else 0) for i in range(n_rows)],
+        dtype=float,
+        columns=["Z"],
+        index=None,
+    )
+    result = g.shift(-1, fill_value=0)[["Z"]]
+
+    tm.assert_frame_equal(result, expected)
+
+
+def test_group_shift_lose_timezone():
+    # GH 30134
+    now_dt = Timestamp.utcnow()
+    df = DataFrame({"a": [1, 1], "date": now_dt})
+    result = df.groupby("a").shift(0).iloc[0]
+    expected = Series({"date": now_dt}, name=result.name)
+    tm.assert_series_equal(result, expected)
+
+
+def test_group_diff_real(any_real_dtype):
+    df = DataFrame({"a": [1, 2, 3, 3, 2], "b": [1, 2, 3, 4, 5]}, dtype=any_real_dtype)
+    result = df.groupby("a")["b"].diff()
+    exp_dtype = "float"
+    if any_real_dtype in ["int8", "int16", "float32"]:
+        exp_dtype = "float32"
+    expected = Series([np.nan, np.nan, np.nan, 1.0, 3.0], dtype=exp_dtype, name="b")
+    tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+    "data",
+    [
+        [
+            Timestamp("2013-01-01"),
+            Timestamp("2013-01-02"),
+            Timestamp("2013-01-03"),
+        ],
+        [Timedelta("5 days"), Timedelta("6 days"), Timedelta("7 days")],
+    ],
+)
+def test_group_diff_datetimelike(data):
+    df = DataFrame({"a": [1, 2, 2], "b": data})
+    result = df.groupby("a")["b"].diff()
+    expected = Series([NaT, NaT, Timedelta("1 days")], name="b")
+    tm.assert_series_equal(result, expected)
+
+
+def test_group_diff_bool():
+    df = DataFrame({"a": [1, 2, 3, 3, 2], "b": [True, True, False, False, True]})
+    result = df.groupby("a")["b"].diff()
+    expected = Series([np.nan, np.nan, np.nan, False, False], name="b")
+    tm.assert_series_equal(result, expected)
+
+
+def test_group_diff_object_raises(object_dtype):
+    df = DataFrame(
+        {"a": ["foo", "bar", "bar"], "b": ["baz", "foo", "foo"]}, dtype=object_dtype
+    )
+    with pytest.raises(TypeError, match=r"unsupported operand type\(s\) for -"):
+        df.groupby("a")["b"].diff()

pandas/tests/test_algos.py

@@ -2409,3 +2409,10 @@ def test_diff_ea_axis(self):
         msg = "cannot diff DatetimeArray on axis=1"
         with pytest.raises(ValueError, match=msg):
             algos.diff(dta, 1, axis=1)
+
+    @pytest.mark.parametrize("dtype", ["int8", "int16"])
+    def test_diff_low_precision_int(self, dtype):
+        arr = np.array([0, 1, 1, 0, 0], dtype=dtype)
+        result = algos.diff(arr, 1)
+        expected = np.array([np.nan, 1, 0, -1, 0], dtype="float32")
+        tm.assert_numpy_array_equal(result, expected)