BUG: groupby.shift returns different columns when fill_value is specified

doc/source/whatsnew/v1.4.0.rst

@@ -187,7 +187,7 @@ Plotting
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
--
+- Bug in :meth:`GroupBy.shift` that would return different columns if ``fill_value`` was not None (:issue:`41556`)
 -
 
 Reshaping

pandas/core/groupby/groupby.py

@@ -2780,6 +2780,7 @@ def _get_cythonized_result(
         result_is_index: bool = False,
         pre_processing=None,
         post_processing=None,
+        fill_value=None,
         **kwargs,
     ):
         """
@@ -2830,6 +2831,8 @@ def _get_cythonized_result(
             second argument, i.e. the signature should be
             (ndarray, Type). If `needs_nullable=True`, a third argument should be
             `nullable`, to allow for processing specific to nullable values.
+        fill_value : any, default None
+            The scalar value to use for newly introduced missing values.
         **kwargs : dict
             Extra arguments to be passed back to Cython funcs
 
@@ -2928,7 +2931,7 @@ def _get_cythonized_result(
                 result = result.reshape(-1)
 
             if result_is_index:
-                result = algorithms.take_nd(values, result)
+                result = algorithms.take_nd(values, result, fill_value=fill_value)
 
             if post_processing:
                 result = post_processing(result, inferences)
@@ -2975,7 +2978,7 @@ def shift(self, periods=1, freq=None, axis=0, fill_value=None):
         tshift : Shift the time index, using the index’s frequency
             if available.
         """
-        if freq is not None or axis != 0 or not isna(fill_value):
+        if freq is not None or axis != 0:
             return self.apply(lambda x: x.shift(periods, freq, axis, fill_value))
 
         return self._get_cythonized_result(
@@ -2985,6 +2988,7 @@ def shift(self, periods=1, freq=None, axis=0, fill_value=None):
             needs_ngroups=True,
             result_is_index=True,
             periods=periods,
+            fill_value=fill_value,
         )
 
     @final

pandas/tests/groupby/test_groupby_shift_diff.py

@@ -55,7 +55,7 @@ def test_group_shift_with_fill_value():
         columns=["Z"],
         index=None,
     )
-    result = g.shift(-1, fill_value=0)[["Z"]]
+    result = g.shift(-1, fill_value=0)
 
     tm.assert_frame_equal(result, expected)