REF: Add Manager.column_setitem to set values into a single column (without intermediate series)

Author: jorisvandenbossche

pandas/core/frame.py

@@ -3924,16 +3924,16 @@ def _set_value(
             Sets whether or not index/col interpreted as indexers
         """
         try:
-            if takeable:
-                series = self._ixs(col, axis=1)
-                loc = index
-            else:
-                series = self._get_item_cache(col)
-                loc = self.index.get_loc(index)
-
             # setitem_inplace will do validation that may raise TypeError,
             #  ValueError, or LossySetitemError
-            series._mgr.setitem_inplace(loc, value)
+            # breakpoint()
+            if takeable:
+                self._mgr.column_setitem(col, index, value)
+            else:
+                icol = self.columns.get_loc(col)
+                index = self.index.get_loc(index)
+                self._mgr.column_setitem(icol, index, value)
+            self._clear_item_cache()
 
         except (KeyError, TypeError, ValueError, LossySetitemError):
             # set using a non-recursive method & reset the cache

pandas/core/indexing.py

@@ -52,7 +52,6 @@
 from pandas.core.indexers import (
     check_array_indexer,
     is_empty_indexer,
-    is_exact_shape_match,
     is_list_like_indexer,
     is_scalar_indexer,
     length_of_indexer,
@@ -1936,42 +1935,31 @@ def _setitem_single_column(self, loc: int, value, plane_indexer):
         """
         pi = plane_indexer
 
-        ser = self.obj._ixs(loc, axis=1)
-
         # perform the equivalent of a setitem on the info axis
         # as we have a null slice or a slice with full bounds
         # which means essentially reassign to the columns of a
         # multi-dim object
         # GH#6149 (null slice), GH#10408 (full bounds)
         if com.is_null_slice(pi) or com.is_full_slice(pi, len(self.obj)):
-            ser = value
+            self.obj._iset_item(loc, value)
         elif (
             is_array_like(value)
-            and is_exact_shape_match(ser, value)
+            and len(value.shape) > 0
+            and self.obj.shape[0] == value.shape[0]
             and not is_empty_indexer(pi)
         ):
             if is_list_like(pi):
-                ser = value[np.argsort(pi)]
+                value = value[np.argsort(pi)]
             else:
                 # in case of slice
-                ser = value[pi]
+                value = value[pi]
+            self.obj._iset_item(loc, value)
         else:
             # set the item, first attempting to operate inplace, then
             #  falling back to casting if necessary; see
             #  _whatsnew_130.notable_bug_fixes.setitem_column_try_inplace
-
-            orig_values = ser._values
-            ser._mgr = ser._mgr.setitem((pi,), value)
-
-            if ser._values is orig_values:
-                # The setitem happened inplace, so the DataFrame's values
-                #  were modified inplace.
-                return
-            self.obj._iset_item(loc, ser)
-            return
-
-        # reset the sliced object if unique
-        self.obj._iset_item(loc, ser)
+            self.obj._mgr.column_setitem(loc, plane_indexer, value)
+            self.obj._clear_item_cache()
 
     def _setitem_single_block(self, indexer, value, name: str):
         """

pandas/core/internals/array_manager.py

@@ -869,6 +869,20 @@ def iset(
             self.arrays[mgr_idx] = value_arr
         return
 
+    def column_setitem(self, loc: int, idx: int | slice | np.ndarray, value) -> None:
+        """
+        Set values ("setitem") into a single column (not setting the full column).
+
+        This is a method on the ArrayManager level, to avoid creating an
+        intermediate Series at the DataFrame level (`s = df[loc]; s[idx] = value`)
+        """
+        arr = self.arrays[loc]
+        # create temporary SingleArrayManager without ref to use setitem implementation
+        mgr = SingleArrayManager([arr], [self._axes[0]])
+        new_mgr = mgr.setitem((idx,), value)
+        # update existing ArrayManager in-place
+        self.arrays[loc] = new_mgr.arrays[0]
+
     def insert(self, loc: int, item: Hashable, value: ArrayLike) -> None:
         """
         Insert item at selected position.

pandas/core/internals/managers.py

@@ -1185,6 +1185,17 @@ def _iset_single(
         self.blocks = new_blocks
         return
 
+    def column_setitem(self, loc: int, idx: int | slice | np.ndarray, value) -> None:
+        """
+        Set values ("setitem") into a single column (not setting the full column).
+
+        This is a method on the BlockManager level, to avoid creating an
+        intermediate Series at the DataFrame level (`s = df[loc]; s[idx] = value`)
+        """
+        col_mgr = self.iget(loc)
+        new_mgr = col_mgr.setitem((idx,), value)
+        self.iset(loc, new_mgr._block.values, inplace=True)
+
     def insert(self, loc: int, item: Hashable, value: ArrayLike) -> None:
         """
         Insert item at selected position.

pandas/tests/frame/indexing/test_setitem.py

@@ -1082,7 +1082,7 @@ def test_setitem_partial_column_inplace(self, consolidate, using_array_manager):
         tm.assert_numpy_array_equal(zvals, expected.values)
         assert np.shares_memory(zvals, df["z"]._values)
         if not consolidate:
-            assert df["z"]._values is zvals
+            assert df["z"]._values.base is zvals.base
 
     def test_setitem_duplicate_columns_not_inplace(self):
         # GH#39510

pandas/tests/indexing/test_partial.py

@@ -266,7 +266,7 @@ def test_partial_setting(self):
         with pytest.raises(IndexError, match=msg):
             s.iat[3] = 5.0
 
-    def test_partial_setting_frame(self):
+    def test_partial_setting_frame(self, using_array_manager):
         df_orig = DataFrame(
             np.arange(6).reshape(3, 2), columns=["A", "B"], dtype="int64"
         )
@@ -279,6 +279,8 @@ def test_partial_setting_frame(self):
             df.iloc[4, 2] = 5.0
 
         msg = "index 2 is out of bounds for axis 0 with size 2"
+        if using_array_manager:
+            msg = "list index out of range"
         with pytest.raises(IndexError, match=msg):
             df.iat[4, 2] = 5.0