Merge branch 'main' into numeric_to_df_cum

Author: Khor Chean Wei

asv_bench/asv.conf.json

@@ -41,6 +41,7 @@
     // pip (with all the conda available packages installed first,
     // followed by the pip installed packages).
     "matrix": {
+        "pip+build": [],
         "Cython": ["3.0"],
         "matplotlib": [],
         "sqlalchemy": [],

ci/deps/actions-310.yaml

@@ -24,6 +24,8 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
+  # https://github.com/conda-forge/pytables-feedstock/issues/97
+  - c-blosc2=2.13.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2023.10.0

ci/deps/actions-311-downstream_compat.yaml

@@ -26,6 +26,8 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
+  # https://github.com/conda-forge/pytables-feedstock/issues/97
+  - c-blosc2=2.13.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2023.10.0

ci/deps/actions-311.yaml

@@ -24,6 +24,8 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
+  # https://github.com/conda-forge/pytables-feedstock/issues/97
+  - c-blosc2=2.13.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2023.10.0

ci/deps/actions-312.yaml

@@ -24,6 +24,8 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
+  # https://github.com/conda-forge/pytables-feedstock/issues/97
+  - c-blosc2=2.13.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2023.10.0

ci/deps/actions-39-minimum_versions.yaml

@@ -27,6 +27,8 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4=4.11.2
+  # https://github.com/conda-forge/pytables-feedstock/issues/97
+  - c-blosc2=2.13.2
   - blosc=1.21.3
   - bottleneck=1.3.6
   - fastparquet=2023.10.0

ci/deps/actions-39.yaml

@@ -24,6 +24,8 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
+  # https://github.com/conda-forge/pytables-feedstock/issues/97
+  - c-blosc2=2.13.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2023.10.0

ci/deps/circle-310-arm64.yaml

@@ -25,6 +25,8 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
+  # https://github.com/conda-forge/pytables-feedstock/issues/97
+  - c-blosc2=2.13.2
   - blosc>=1.21.3
   - bottleneck>=1.3.6
   - fastparquet>=2023.10.0

doc/source/user_guide/basics.rst

@@ -476,15 +476,15 @@ For example:
 .. ipython:: python
 
    df
-   df.mean(0)
-   df.mean(1)
+   df.mean(axis=0)
+   df.mean(axis=1)
 
 All such methods have a ``skipna`` option signaling whether to exclude missing
 data (``True`` by default):
 
 .. ipython:: python
 
-   df.sum(0, skipna=False)
+   df.sum(axis=0, skipna=False)
    df.sum(axis=1, skipna=True)
 
 Combined with the broadcasting / arithmetic behavior, one can describe various
@@ -495,8 +495,8 @@ standard deviation of 1), very concisely:
 
    ts_stand = (df - df.mean()) / df.std()
    ts_stand.std()
-   xs_stand = df.sub(df.mean(1), axis=0).div(df.std(1), axis=0)
-   xs_stand.std(1)
+   xs_stand = df.sub(df.mean(axis=1), axis=0).div(df.std(axis=1), axis=0)
+   xs_stand.std(axis=1)
 
 Note that methods like :meth:`~DataFrame.cumsum` and :meth:`~DataFrame.cumprod`
 preserve the location of ``NaN`` values. This is somewhat different from

doc/source/user_guide/indexing.rst

@@ -952,7 +952,7 @@ To select a row where each column meets its own criterion:
 
   values = {'ids': ['a', 'b'], 'ids2': ['a', 'c'], 'vals': [1, 3]}
 
-  row_mask = df.isin(values).all(1)
+  row_mask = df.isin(values).all(axis=1)
 
   df[row_mask]
 

doc/source/whatsnew/v2.2.1.rst

@@ -87,4 +87,4 @@ Other
 Contributors
 ~~~~~~~~~~~~
 
-.. contributors:: v2.2.0..v2.2.1|HEAD
+.. contributors:: v2.2.0..v2.2.1

doc/source/whatsnew/v2.2.2.rst

@@ -1,6 +1,6 @@
 .. _whatsnew_222:
 
-What's new in 2.2.2 (April XX, 2024)
+What's new in 2.2.2 (April 10, 2024)
 ---------------------------------------
 
 These are the changes in pandas 2.2.2. See :ref:`release` for a full changelog
@@ -9,6 +9,21 @@ including other versions of pandas.
 {{ header }}
 
 .. ---------------------------------------------------------------------------
+
+.. _whatsnew_220.np2_compat:
+
+Pandas 2.2.2 is now compatible with numpy 2.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Pandas 2.2.2 is the first version of pandas that is generally compatible with the upcoming
+numpy 2.0 release, and wheels for pandas 2.2.2 will work with both numpy 1.x and 2.x.
+
+One major caveat is that arrays created with numpy 2.0's new ``StringDtype`` will convert
+to ``object`` dtyped arrays upon :class:`Series`/:class:`DataFrame` creation.
+Full support for numpy 2.0's StringDtype is expected to land in pandas 3.0.
+
+As usual please report any bugs discovered to our `issue tracker <https://github.com/pandas-dev/pandas/issues/new/choose>`_
+
 .. _whatsnew_222.regressions:
 
 Fixed regressions
@@ -40,3 +55,5 @@ Other
 
 Contributors
 ~~~~~~~~~~~~
+
+.. contributors:: v2.2.1..v2.2.2|HEAD

doc/source/whatsnew/v3.0.0.rst

@@ -33,6 +33,7 @@ Other enhancements
 - :meth:`Styler.set_tooltips` provides alternative method to storing tooltips by using title attribute of td elements. (:issue:`56981`)
 - Allow dictionaries to be passed to :meth:`pandas.Series.str.replace` via ``pat`` parameter (:issue:`51748`)
 - Support passing a :class:`Series` input to :func:`json_normalize` that retains the :class:`Series` :class:`Index` (:issue:`51452`)
+- Support reading value labels from Stata 108-format (Stata 6) and earlier files (:issue:`58154`)
 - Users can globally disable any ``PerformanceWarning`` by setting the option ``mode.performance_warnings`` to ``False`` (:issue:`56920`)
 - :meth:`Styler.format_index_names` can now be used to format the index and column names (:issue:`48936` and :issue:`47489`)
 - :meth:`DataFrame.cum*` methods now have a ``numeric_only`` parameter (:issue:`53072`)
@@ -190,6 +191,7 @@ Other Deprecations
 
 - Deprecated :meth:`Timestamp.utcfromtimestamp`, use ``Timestamp.fromtimestamp(ts, "UTC")`` instead (:issue:`56680`)
 - Deprecated :meth:`Timestamp.utcnow`, use ``Timestamp.now("UTC")`` instead (:issue:`56680`)
+- Deprecated allowing non-keyword arguments in :meth:`DataFrame.all`, :meth:`DataFrame.min`, :meth:`DataFrame.max`, :meth:`DataFrame.sum`, :meth:`DataFrame.prod`, :meth:`DataFrame.mean`, :meth:`DataFrame.median`, :meth:`DataFrame.sem`, :meth:`DataFrame.var`, :meth:`DataFrame.std`, :meth:`DataFrame.skew`, :meth:`DataFrame.kurt`, :meth:`Series.all`,  :meth:`Series.min`, :meth:`Series.max`, :meth:`Series.sum`, :meth:`Series.prod`, :meth:`Series.mean`, :meth:`Series.median`, :meth:`Series.sem`, :meth:`Series.var`, :meth:`Series.std`, :meth:`Series.skew`, and :meth:`Series.kurt`. (:issue:`57087`)
 - Deprecated allowing non-keyword arguments in :meth:`Series.to_markdown` except ``buf``. (:issue:`57280`)
 - Deprecated allowing non-keyword arguments in :meth:`Series.to_string` except ``buf``. (:issue:`57280`)
 -
@@ -347,6 +349,7 @@ Bug fixes
 - Fixed bug in :meth:`Series.diff` allowing non-integer values for the ``periods`` argument. (:issue:`56607`)
 - Fixed bug in :meth:`Series.rank` that doesn't preserve missing values for nullable integers when ``na_option='keep'``. (:issue:`56976`)
 - Fixed bug in :meth:`Series.replace` and :meth:`DataFrame.replace` inconsistently replacing matching instances when ``regex=True`` and missing values are present. (:issue:`56599`)
+- Fixed bug in :meth:`read_csv` raising ``TypeError`` when ``index_col`` is specified and ``na_values`` is a dict containing the key ``None``. (:issue:`57547`)
 
 Categorical
 ^^^^^^^^^^^

environment.yml

@@ -28,6 +28,8 @@ dependencies:
 
   # optional dependencies
   - beautifulsoup4>=4.11.2
+  # https://github.com/conda-forge/pytables-feedstock/issues/97
+  - c-blosc2=2.13.2
   - blosc
   - bottleneck>=1.3.6
   - fastparquet>=2023.10.0

pandas/_libs/tslib.pyx

@@ -70,7 +70,6 @@ from pandas._libs.tslibs.conversion cimport (
 from pandas._libs.tslibs.dtypes cimport npy_unit_to_abbrev
 from pandas._libs.tslibs.nattype cimport (
     NPY_NAT,
-    c_NaT as NaT,
     c_nat_strings as nat_strings,
 )
 from pandas._libs.tslibs.timestamps cimport _Timestamp
@@ -346,39 +345,6 @@ def array_with_unit_to_datetime(
     return result, tz
 
 
-cdef _array_with_unit_to_datetime_object_fallback(ndarray[object] values, str unit):
-    cdef:
-        Py_ssize_t i, n = len(values)
-        ndarray[object] oresult
-        tzinfo tz = None
-
-    # TODO: fix subtle differences between this and no-unit code
-    oresult = cnp.PyArray_EMPTY(values.ndim, values.shape, cnp.NPY_OBJECT, 0)
-    for i in range(n):
-        val = values[i]
-
-        if checknull_with_nat_and_na(val):
-            oresult[i] = <object>NaT
-        elif is_integer_object(val) or is_float_object(val):
-
-            if val != val or val == NPY_NAT:
-                oresult[i] = <object>NaT
-            else:
-                try:
-                    oresult[i] = Timestamp(val, unit=unit)
-                except OutOfBoundsDatetime:
-                    oresult[i] = val
-
-        elif isinstance(val, str):
-            if len(val) == 0 or val in nat_strings:
-                oresult[i] = <object>NaT
-
-            else:
-                oresult[i] = val
-
-    return oresult, tz
-
-
 @cython.wraparound(False)
 @cython.boundscheck(False)
 def first_non_null(values: ndarray) -> int:

pandas/core/apply.py

@@ -1710,9 +1710,9 @@ def normalize_keyword_aggregation(
     # TODO: aggspec type: typing.Dict[str, List[AggScalar]]
     aggspec = defaultdict(list)
     order = []
-    columns, pairs = list(zip(*kwargs.items()))
+    columns = tuple(kwargs.keys())
 
-    for column, aggfunc in pairs:
+    for column, aggfunc in kwargs.values():
         aggspec[column].append(aggfunc)
         order.append((column, com.get_callable_name(aggfunc) or aggfunc))
 

pandas/core/frame.py

@@ -64,6 +64,7 @@
 from pandas.util._decorators import (
     Appender,
     Substitution,
+    deprecate_nonkeyword_arguments,
     doc,
     set_module,
 )
@@ -6167,12 +6168,13 @@ class    max    type
             names = self.index._get_default_index_names(names, default)
 
             if isinstance(self.index, MultiIndex):
-                to_insert = zip(self.index.levels, self.index.codes)
+                to_insert = zip(reversed(self.index.levels), reversed(self.index.codes))
             else:
                 to_insert = ((self.index, None),)
 
             multi_col = isinstance(self.columns, MultiIndex)
-            for i, (lev, lab) in reversed(list(enumerate(to_insert))):
+            for j, (lev, lab) in enumerate(to_insert, start=1):
+                i = self.index.nlevels - j
                 if level is not None and i not in level:
                     continue
                 name = names[i]
@@ -11543,6 +11545,7 @@ def all(
         **kwargs,
     ) -> Series | bool: ...
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="all")
     @doc(make_doc("all", ndim=1))
     def all(
         self,
@@ -11589,6 +11592,7 @@ def min(
         **kwargs,
     ) -> Series | Any: ...
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="min")
     @doc(make_doc("min", ndim=2))
     def min(
         self,
@@ -11635,6 +11639,7 @@ def max(
         **kwargs,
     ) -> Series | Any: ...
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="max")
     @doc(make_doc("max", ndim=2))
     def max(
         self,
@@ -11650,6 +11655,7 @@ def max(
             result = result.__finalize__(self, method="max")
         return result
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="sum")
     @doc(make_doc("sum", ndim=2))
     def sum(
         self,
@@ -11670,6 +11676,7 @@ def sum(
             result = result.__finalize__(self, method="sum")
         return result
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="prod")
     @doc(make_doc("prod", ndim=2))
     def prod(
         self,
@@ -11721,6 +11728,7 @@ def mean(
         **kwargs,
     ) -> Series | Any: ...
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="mean")
     @doc(make_doc("mean", ndim=2))
     def mean(
         self,
@@ -11767,6 +11775,7 @@ def median(
         **kwargs,
     ) -> Series | Any: ...
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="median")
     @doc(make_doc("median", ndim=2))
     def median(
         self,
@@ -11816,6 +11825,7 @@ def sem(
         **kwargs,
     ) -> Series | Any: ...
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="sem")
     @doc(make_doc("sem", ndim=2))
     def sem(
         self,
@@ -11866,6 +11876,7 @@ def var(
         **kwargs,
     ) -> Series | Any: ...
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="var")
     @doc(make_doc("var", ndim=2))
     def var(
         self,
@@ -11916,6 +11927,7 @@ def std(
         **kwargs,
     ) -> Series | Any: ...
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="std")
     @doc(make_doc("std", ndim=2))
     def std(
         self,
@@ -11963,6 +11975,7 @@ def skew(
         **kwargs,
     ) -> Series | Any: ...
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="skew")
     @doc(make_doc("skew", ndim=2))
     def skew(
         self,
@@ -12009,6 +12022,7 @@ def kurt(
         **kwargs,
     ) -> Series | Any: ...
 
+    @deprecate_nonkeyword_arguments(version="3.0", allowed_args=["self"], name="kurt")
     @doc(make_doc("kurt", ndim=2))
     def kurt(
         self,

pandas/core/groupby/grouper.py

@@ -263,7 +263,6 @@ def __init__(
         self.sort = sort
         self.dropna = dropna
 
-        self._grouper_deprecated = None
         self._indexer_deprecated: npt.NDArray[np.intp] | None = None
         self.binner = None
         self._grouper = None
@@ -292,10 +291,6 @@ def _get_grouper(
             validate=validate,
             dropna=self.dropna,
         )
-        # Without setting this, subsequent lookups to .groups raise
-        # error: Incompatible types in assignment (expression has type "BaseGrouper",
-        # variable has type "None")
-        self._grouper_deprecated = grouper  # type: ignore[assignment]
 
         return grouper, obj
 

pandas/core/groupby/ops.py

@@ -706,7 +706,7 @@ def groups(self) -> dict[Hashable, Index]:
             return self.groupings[0].groups
         result_index, ids = self.result_index_and_ids
         values = result_index._values
-        categories = Categorical(ids, categories=np.arange(len(result_index)))
+        categories = Categorical(ids, categories=range(len(result_index)))
         result = {
             # mypy is not aware that group has to be an integer
             values[group]: self.axis.take(axis_ilocs)  # type: ignore[call-overload]