Merge branch 'main' into np-array-copy-keyword

Author: mtsokol

doc/source/whatsnew/v3.0.0.rst

@@ -244,6 +244,7 @@ Removal of prior version deprecations/changes
 
 Performance improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
+- :meth:`Series.str.extract` returns a :class:`RangeIndex` columns instead of an :class:`Index` column when possible (:issue:`57542`)
 - Performance improvement in :class:`DataFrame` when ``data`` is a ``dict`` and ``columns`` is specified (:issue:`24368`)
 - Performance improvement in :meth:`DataFrame.join` for sorted but non-unique indexes (:issue:`56941`)
 - Performance improvement in :meth:`DataFrame.join` when left and/or right are non-unique and ``how`` is ``"left"``, ``"right"``, or ``"inner"`` (:issue:`56817`)
@@ -252,11 +253,11 @@ Performance improvements
 - Performance improvement in :meth:`Index.join` by propagating cached attributes in cases where the result matches one of the inputs (:issue:`57023`)
 - Performance improvement in :meth:`Index.take` when ``indices`` is a full range indexer from zero to length of index (:issue:`56806`)
 - Performance improvement in :meth:`MultiIndex.equals` for equal length indexes (:issue:`56990`)
+- Performance improvement in :meth:`RangeIndex.__getitem__` with a boolean mask returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57588`)
 - Performance improvement in :meth:`RangeIndex.append` when appending the same index (:issue:`57252`)
 - Performance improvement in :meth:`RangeIndex.take` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57445`)
-- Performance improvement in indexing operations for string dtypes (:issue:`56997`)
-- :meth:`Series.str.extract` returns a :class:`RangeIndex` columns instead of an :class:`Index` column when possible (:issue:`?``)
 - Performance improvement in ``DataFrameGroupBy.__len__`` and ``SeriesGroupBy.__len__`` (:issue:`57595`)
+- Performance improvement in indexing operations for string dtypes (:issue:`56997`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.bug_fixes:
@@ -265,6 +266,7 @@ Bug fixes
 ~~~~~~~~~
 - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`)
 - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`)
+- Fixed bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`)
 - Fixed bug in :meth:`Series.diff` allowing non-integer values for the ``periods`` argument. (:issue:`56607`)
 
 Categorical
@@ -324,6 +326,7 @@ MultiIndex
 
 I/O
 ^^^
+- Bug in :meth:`DataFrame.to_excel` when writing empty :class:`DataFrame` with :class:`MultiIndex` on both axes (:issue:`57696`)
 -
 -
 

pandas/_libs/src/vendored/ujson/python/objToJSON.c

@@ -74,7 +74,6 @@ typedef struct __NpyArrContext {
   npy_intp ndim;
   npy_intp index[NPY_MAXDIMS];
   int type_num;
-  PyArray_GetItemFunc *getitem;
 
   char **rowLabels;
   char **columnLabels;
@@ -405,7 +404,6 @@ static void NpyArr_iterBegin(JSOBJ _obj, JSONTypeContext *tc) {
   }
 
   npyarr->array = (PyObject *)obj;
-  npyarr->getitem = (PyArray_GetItemFunc *)PyArray_DESCR(obj)->f->getitem;
   npyarr->dataptr = PyArray_DATA(obj);
   npyarr->ndim = PyArray_NDIM(obj) - 1;
   npyarr->curdim = 0;
@@ -492,7 +490,7 @@ static int NpyArr_iterNextItem(JSOBJ obj, JSONTypeContext *tc) {
     ((PyObjectEncoder *)tc->encoder)->npyValue = npyarr->dataptr;
     ((PyObjectEncoder *)tc->encoder)->npyCtxtPassthru = npyarr;
   } else {
-    GET_TC(tc)->itemValue = npyarr->getitem(npyarr->dataptr, npyarr->array);
+    GET_TC(tc)->itemValue = PyArray_GETITEM(arrayobj, npyarr->dataptr);
   }
 
   npyarr->dataptr += npyarr->stride;

pandas/core/array_algos/quantile.py

@@ -4,6 +4,8 @@
 
 import numpy as np
 
+from pandas.compat.numpy import np_version_gt2
+
 from pandas.core.dtypes.missing import (
     isna,
     na_value_for_dtype,
@@ -102,7 +104,8 @@ def quantile_with_mask(
             interpolation=interpolation,
         )
 
-        result = np.array(result, copy=False)
+        copy_false = None if np_version_gt2 else False
+        result = np.array(result, copy=copy_false)
         result = result.T
 
     return result
@@ -199,11 +202,12 @@ def _nanpercentile(
             _nanpercentile_1d(val, m, qs, na_value, interpolation=interpolation)
             for (val, m) in zip(list(values), list(mask))
         ]
+        copy_false = None if np_version_gt2 else False
         if values.dtype.kind == "f":
             # preserve itemsize
-            result = np.array(result, dtype=values.dtype, copy=False).T
+            result = np.array(result, dtype=values.dtype, copy=copy_false).T
         else:
-            result = np.array(result, copy=False).T
+            result = np.array(result, copy=copy_false).T
             if (
                 result.dtype != values.dtype
                 and not mask.all()

pandas/core/arrays/arrow/array.py

@@ -659,7 +659,9 @@ def __arrow_array__(self, type=None):
         """Convert myself to a pyarrow ChunkedArray."""
         return self._pa_array
 
-    def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+    def __array__(
+        self, dtype: NpDtype | None = None, copy: bool | None = None
+    ) -> np.ndarray:
         """Correctly construct numpy arrays when passed to `np.asarray()`."""
         return self.to_numpy(dtype=dtype)
 

pandas/core/arrays/base.py

@@ -27,7 +27,10 @@
     lib,
 )
 from pandas.compat import set_function_name
-from pandas.compat.numpy import function as nv
+from pandas.compat.numpy import (
+    function as nv,
+    np_version_gt2,
+)
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import (
     Appender,
@@ -710,6 +713,8 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
                 return self
             else:
                 return self.copy()
+        if np_version_gt2 and not copy:
+            copy = None
 
         if isinstance(dtype, ExtensionDtype):
             cls = dtype.construct_array_type()

pandas/core/arrays/categorical.py

@@ -1659,7 +1659,9 @@ def _validate_codes_for_dtype(cls, codes, *, dtype: CategoricalDtype) -> np.ndar
     # -------------------------------------------------------------
 
     @ravel_compat
-    def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+    def __array__(
+        self, dtype: NpDtype | None = None, copy: bool | None = None
+    ) -> np.ndarray:
         """
         The numpy array interface.
 

pandas/core/arrays/datetimelike.py

@@ -353,7 +353,9 @@ def _formatter(self, boxed: bool = False) -> Callable[[object], str]:
     # ----------------------------------------------------------------
     # Array-Like / EA-Interface Methods
 
-    def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+    def __array__(
+        self, dtype: NpDtype | None = None, copy: bool | None = None
+    ) -> np.ndarray:
         # used for Timedelta/DatetimeArray, overwritten by PeriodArray
         if is_object_dtype(dtype):
             return np.array(list(self), dtype=object)

pandas/core/arrays/datetimes.py

@@ -43,6 +43,7 @@
     tzconversion,
 )
 from pandas._libs.tslibs.dtypes import abbrev_to_npy_unit
+from pandas.compat.numpy import np_version_gt2
 from pandas.errors import PerformanceWarning
 from pandas.util._exceptions import find_stack_level
 from pandas.util._validators import validate_inclusive
@@ -649,12 +650,12 @@ def _resolution_obj(self) -> Resolution:
     # ----------------------------------------------------------------
     # Array-Like / EA-Interface Methods
 
-    def __array__(self, dtype=None) -> np.ndarray:
+    def __array__(self, dtype=None, copy=None) -> np.ndarray:
         if dtype is None and self.tz:
             # The default for tz-aware is object, to preserve tz info
             dtype = object
 
-        return super().__array__(dtype=dtype)
+        return super().__array__(dtype=dtype, copy=copy)
 
     def __iter__(self) -> Iterator:
         """
@@ -2421,7 +2422,8 @@ def objects_to_datetime64(
     assert errors in ["raise", "coerce"]
 
     # if str-dtype, convert
-    data = np.array(data, copy=False, dtype=np.object_)
+    copy_false = None if np_version_gt2 else False
+    data = np.array(data, dtype=np.object_, copy=copy_false)
 
     result, tz_parsed = tslib.array_to_datetime(
         data,

pandas/core/arrays/interval.py

@@ -1564,7 +1564,9 @@ def is_non_overlapping_monotonic(self) -> bool:
     # ---------------------------------------------------------------------
     # Conversion
 
-    def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+    def __array__(
+        self, dtype: NpDtype | None = None, copy: bool | None = None
+    ) -> np.ndarray:
         """
         Return the IntervalArray's data as a numpy array of Interval
         objects (with dtype='object')

pandas/core/arrays/masked.py

@@ -594,7 +594,9 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
 
     __array_priority__ = 1000  # higher than ndarray so ops dispatch to us
 
-    def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+    def __array__(
+        self, dtype: NpDtype | None = None, copy: bool | None = None
+    ) -> np.ndarray:
         """
         the array interface, return my values
         We return an object array here to preserve our scalar values

pandas/core/arrays/numeric.py

@@ -13,6 +13,7 @@
     lib,
     missing as libmissing,
 )
+from pandas.compat.numpy import np_version_gt2
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import cache_readonly
 
@@ -137,6 +138,12 @@ def _coerce_to_data_and_mask(
     values, dtype, copy: bool, dtype_cls: type[NumericDtype], default_dtype: np.dtype
 ):
     checker = dtype_cls._checker
+    if np_version_gt2:
+        copy_false = None
+        if not copy:
+            copy = None
+    else:
+        copy_false = False
 
     mask = None
     inferred_type = None
@@ -208,9 +215,9 @@ def _coerce_to_data_and_mask(
                     inferred_type not in ["floating", "mixed-integer-float"]
                     and not mask.any()
                 ):
-                    values = np.array(original, dtype=dtype, copy=False)
+                    values = np.array(original, dtype=dtype, copy=copy_false)
                 else:
-                    values = np.array(original, dtype="object", copy=False)
+                    values = np.array(original, dtype="object", copy=copy_false)
 
     # we copy as need to coerce here
     if mask.any():

pandas/core/arrays/numpy_.py

@@ -150,7 +150,9 @@ def dtype(self) -> NumpyEADtype:
     # ------------------------------------------------------------------------
     # NumPy Array Interface
 
-    def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+    def __array__(
+        self, dtype: NpDtype | None = None, copy: bool | None = None
+    ) -> np.ndarray:
         return np.asarray(self._ndarray, dtype=dtype)
 
     def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):

pandas/core/arrays/period.py

@@ -50,6 +50,7 @@
     get_period_field_arr,
     period_asfreq_arr,
 )
+from pandas.compat.numpy import np_version_gt2
 from pandas.util._decorators import (
     cache_readonly,
     doc,
@@ -243,6 +244,9 @@ def __init__(
             if not isinstance(dtype, PeriodDtype):
                 raise ValueError(f"Invalid dtype {dtype} for PeriodArray")
 
+        if np_version_gt2 and not copy:
+            copy = None
+
         if isinstance(values, ABCSeries):
             values = values._values
             if not isinstance(values, type(self)):
@@ -400,7 +404,9 @@ def freq(self) -> BaseOffset:
     def freqstr(self) -> str:
         return PeriodDtype(self.freq)._freqstr
 
-    def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+    def __array__(
+        self, dtype: NpDtype | None = None, copy: bool | None = None
+    ) -> np.ndarray:
         if dtype == "i8":
             return self.asi8
         elif dtype == bool:

pandas/core/arrays/sparse/array.py

@@ -554,7 +554,9 @@ def from_spmatrix(cls, data: spmatrix) -> Self:
 
         return cls._simple_new(arr, index, dtype)
 
-    def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+    def __array__(
+        self, dtype: NpDtype | None = None, copy: bool | None = None
+    ) -> np.ndarray:
         fill_value = self.fill_value
 
         if self.sp_index.ngaps == 0:

pandas/core/arrays/timedeltas.py

@@ -36,7 +36,10 @@
     parse_timedelta_unit,
     truediv_object_array,
 )
-from pandas.compat.numpy import function as nv
+from pandas.compat.numpy import (
+    function as nv,
+    np_version_gt2,
+)
 from pandas.util._validators import validate_endpoints
 
 from pandas.core.dtypes.common import (
@@ -1072,6 +1075,9 @@ def sequence_to_td64ns(
         # This includes datetime64-dtype, see GH#23539, GH#29794
         raise TypeError(f"dtype {data.dtype} cannot be converted to timedelta64[ns]")
 
+    copy_false = None if np_version_gt2 else False
+    if not copy:
+        copy = copy_false
     data = np.array(data, copy=copy)
 
     assert data.dtype.kind == "m"
@@ -1152,7 +1158,8 @@ def _objects_to_td64ns(
     higher level.
     """
     # coerce Index to np.ndarray, converting string-dtype if necessary
-    values = np.array(data, dtype=np.object_, copy=False)
+    copy_false = None if np_version_gt2 else False
+    values = np.array(data, dtype=np.object_, copy=copy_false)
 
     result = array_to_timedelta64(values, unit=unit, errors=errors)
     return result.view("timedelta64[ns]")

pandas/core/construction.py

@@ -34,6 +34,7 @@
     DtypeObj,
     T,
 )
+from pandas.compat.numpy import np_version_gt2
 from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.base import ExtensionDtype
@@ -626,6 +627,8 @@ def sanitize_array(
 
     elif hasattr(data, "__array__"):
         # e.g. dask array GH#38645
+        if np_version_gt2 and not copy:
+            copy = None
         data = np.array(data, copy=copy)
         return sanitize_array(
             data,
@@ -735,6 +738,9 @@ def _sanitize_str_dtypes(
     """
     Ensure we have a dtype that is supported by pandas.
     """
+    copy_false = None if np_version_gt2 else False
+    if not copy:
+        copy = copy_false
 
     # This is to prevent mixed-type Series getting all casted to
     # NumPy string type, e.g. NaN --> '-1#IND'.
@@ -744,7 +750,7 @@ def _sanitize_str_dtypes(
         # GH#19853: If data is a scalar, result has already the result
         if not lib.is_scalar(data):
             if not np.all(isna(data)):
-                data = np.array(data, dtype=dtype, copy=False)
+                data = np.array(data, dtype=dtype, copy=copy_false)
             result = np.array(data, dtype=object, copy=copy)
     return result
 
@@ -781,6 +787,8 @@ def _try_cast(
     np.ndarray or ExtensionArray
     """
     is_ndarray = isinstance(arr, np.ndarray)
+    if np_version_gt2 and not copy:
+        copy = None
 
     if dtype == object:
         if not is_ndarray:

pandas/core/dtypes/cast.py

@@ -1491,6 +1491,9 @@ def construct_2d_arraylike_from_scalar(
     value: Scalar, length: int, width: int, dtype: np.dtype, copy: bool
 ) -> np.ndarray:
     shape = (length, width)
+    copy_false = None if np_version_gt2 else False
+    if not copy:
+        copy = copy_false
 
     if dtype.kind in "mM":
         value = _maybe_box_and_unbox_datetimelike(value, dtype)
@@ -1652,7 +1655,8 @@ def maybe_cast_to_integer_array(arr: list | np.ndarray, dtype: np.dtype) -> np.n
                         "out-of-bound Python int",
                         DeprecationWarning,
                     )
-                casted = np.array(arr, dtype=dtype, copy=False)
+                copy_false = None if np_version_gt2 else False
+                casted = np.array(arr, dtype=dtype, copy=copy_false)
         else:
             with warnings.catch_warnings():
                 warnings.filterwarnings("ignore", category=RuntimeWarning)