Merge branch 'main' into numeric_to_df_cum

Author: Khor Chean Wei

.github/workflows/code-checks.yml

@@ -85,7 +85,7 @@ jobs:
         echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV
       if: ${{ steps.build.outcome == 'success' && always() }}
 
-    - name: Typing + pylint
+    - name: Typing
       uses: pre-commit/[email protected]
       with:
         extra_args: --verbose --hook-stage manual --all-files

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ ci:
     autofix_prs: false
     autoupdate_schedule: monthly
     # manual stage hooks
-    skip: [pylint, pyright, mypy]
+    skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.3.4
@@ -30,12 +30,6 @@ repos:
         files: ^pandas
         exclude: ^pandas/tests
         args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix]
-    -   id: ruff
-        name: ruff-use-pd_array-in-core
-        alias: ruff-use-pd_array-in-core
-        files: ^pandas/core/
-        exclude: ^pandas/core/api\.py$
-        args: [--select, "ICN001", --exit-non-zero-on-fix]
     -   id: ruff-format
         exclude: ^scripts
 -   repo: https://github.com/jendrikseipp/vulture
@@ -73,25 +67,6 @@ repos:
     -   id: fix-encoding-pragma
         args: [--remove]
     -   id: trailing-whitespace
--   repo: https://github.com/pylint-dev/pylint
-    rev: v3.0.1
-    hooks:
-    -   id: pylint
-        stages: [manual]
-        args: [--load-plugins=pylint.extensions.redefined_loop_name, --fail-on=I0021]
-    -   id: pylint
-        alias: redefined-outer-name
-        name: Redefining name from outer scope
-        files: ^pandas/
-        exclude: |
-            (?x)
-            ^pandas/tests  # keep excluded
-            |/_testing/  # keep excluded
-            |^pandas/util/_test_decorators\.py  # keep excluded
-            |^pandas/_version\.py  # keep excluded
-            |^pandas/conftest\.py  # keep excluded
-        args: [--disable=all, --enable=redefined-outer-name]
-        stages: [manual]
 -   repo: https://github.com/PyCQA/isort
     rev: 5.13.2
     hooks:

doc/source/whatsnew/v3.0.0.rst

@@ -338,6 +338,7 @@ Bug fixes
 - Fixed bug in :class:`SparseDtype` for equal comparison with na fill value. (:issue:`54770`)
 - Fixed bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`)
 - Fixed bug in :meth:`DataFrame.cumsum` which was raising ``IndexError`` if dtype is ``timedelta64[ns]`` (:issue:`57956`)
+- Fixed bug in :meth:`DataFrame.eval` and :meth:`DataFrame.query` which caused an exception when using NumPy attributes via ``@`` notation, e.g., ``df.eval("@np.floor(a)")``. (:issue:`58041`)
 - Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`)
 - Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`)
 - Fixed bug in :meth:`DataFrame.transform` that was returning the wrong order unless the index was monotonically increasing. (:issue:`57069`)

pandas/_libs/tslib.pyx

@@ -74,7 +74,6 @@ from pandas._libs.tslibs.nattype cimport (
     c_nat_strings as nat_strings,
 )
 from pandas._libs.tslibs.timestamps cimport _Timestamp
-from pandas._libs.tslibs.timezones cimport tz_compare
 
 from pandas._libs.tslibs import (
     Resolution,
@@ -452,13 +451,9 @@ cpdef array_to_datetime(
         ndarray[int64_t] iresult
         npy_datetimestruct dts
         bint utc_convert = bool(utc)
-        bint seen_datetime_offset = False
         bint is_raise = errors == "raise"
         bint is_coerce = errors == "coerce"
-        bint is_same_offsets
         _TSObject tsobj
-        float tz_offset
-        set out_tzoffset_vals = set()
         tzinfo tz, tz_out = None
         cnp.flatiter it = cnp.PyArray_IterNew(values)
         NPY_DATETIMEUNIT item_reso
@@ -568,12 +563,12 @@ cpdef array_to_datetime(
                     # dateutil timezone objects cannot be hashed, so
                     # store the UTC offsets in seconds instead
                     nsecs = tz.utcoffset(None).total_seconds()
-                    out_tzoffset_vals.add(nsecs)
-                    seen_datetime_offset = True
+                    state.out_tzoffset_vals.add(nsecs)
+                    state.found_aware_str = True
                 else:
                     # Add a marker for naive string, to track if we are
                     # parsing mixed naive and aware strings
-                    out_tzoffset_vals.add("naive")
+                    state.out_tzoffset_vals.add("naive")
                     state.found_naive_str = True
 
             else:
@@ -588,41 +583,7 @@ cpdef array_to_datetime(
                 raise
             return values, None
 
-    if seen_datetime_offset and not utc_convert:
-        # GH#17697, GH#57275
-        # 1) If all the offsets are equal, return one offset for
-        #    the parsed dates to (maybe) pass to DatetimeIndex
-        # 2) If the offsets are different, then do not force the parsing
-        #    and raise a ValueError: "cannot parse datetimes with
-        #    mixed time zones unless `utc=True`" instead
-        is_same_offsets = len(out_tzoffset_vals) == 1
-        if not is_same_offsets:
-            raise ValueError(
-                "Mixed timezones detected. Pass utc=True in to_datetime "
-                "or tz='UTC' in DatetimeIndex to convert to a common timezone."
-            )
-        elif state.found_naive or state.found_other:
-            # e.g. test_to_datetime_mixed_awareness_mixed_types
-            raise ValueError("Cannot mix tz-aware with tz-naive values")
-        elif tz_out is not None:
-            # GH#55693
-            tz_offset = out_tzoffset_vals.pop()
-            tz_out2 = timezone(timedelta(seconds=tz_offset))
-            if not tz_compare(tz_out, tz_out2):
-                # e.g. test_to_datetime_mixed_tzs_mixed_types
-                raise ValueError(
-                    "Mixed timezones detected. Pass utc=True in to_datetime "
-                    "or tz='UTC' in DatetimeIndex to convert to a common timezone."
-                )
-            # e.g. test_to_datetime_mixed_types_matching_tzs
-        else:
-            tz_offset = out_tzoffset_vals.pop()
-            tz_out = timezone(timedelta(seconds=tz_offset))
-    elif not utc_convert:
-        if tz_out and (state.found_other or state.found_naive_str):
-            # found_other indicates a tz-naive int, float, dt64, or date
-            # e.g. test_to_datetime_mixed_awareness_mixed_types
-            raise ValueError("Cannot mix tz-aware with tz-naive values")
+    tz_out = state.check_for_mixed_inputs(tz_out, utc)
 
     if infer_reso:
         if state.creso_ever_changed:

pandas/_libs/tslibs/__init__.py

@@ -36,7 +36,7 @@
     "is_supported_dtype",
 ]
 
-from pandas._libs.tslibs import dtypes  # pylint: disable=import-self
+from pandas._libs.tslibs import dtypes
 from pandas._libs.tslibs.conversion import localize_pydatetime
 from pandas._libs.tslibs.dtypes import (
     Resolution,

pandas/_libs/tslibs/strptime.pxd

@@ -18,9 +18,12 @@ cdef class DatetimeParseState:
         bint found_tz
         bint found_naive
         bint found_naive_str
+        bint found_aware_str
         bint found_other
         bint creso_ever_changed
         NPY_DATETIMEUNIT creso
+        set out_tzoffset_vals
 
     cdef tzinfo process_datetime(self, datetime dt, tzinfo tz, bint utc_convert)
     cdef bint update_creso(self, NPY_DATETIMEUNIT item_reso) noexcept
+    cdef tzinfo check_for_mixed_inputs(self, tzinfo tz_out, bint utc)

pandas/_libs/tslibs/strptime.pyx

@@ -252,8 +252,11 @@ cdef class DatetimeParseState:
         # found_naive_str refers to a string that was parsed to a timezone-naive
         #  datetime.
         self.found_naive_str = False
+        self.found_aware_str = False
         self.found_other = False
 
+        self.out_tzoffset_vals = set()
+
         self.creso = creso
         self.creso_ever_changed = False
 
@@ -292,6 +295,58 @@ cdef class DatetimeParseState:
                                  "tz-naive values")
         return tz
 
+    cdef tzinfo check_for_mixed_inputs(
+        self,
+        tzinfo tz_out,
+        bint utc,
+    ):
+        cdef:
+            bint is_same_offsets
+            float tz_offset
+
+        if self.found_aware_str and not utc:
+            # GH#17697, GH#57275
+            # 1) If all the offsets are equal, return one offset for
+            #    the parsed dates to (maybe) pass to DatetimeIndex
+            # 2) If the offsets are different, then do not force the parsing
+            #    and raise a ValueError: "cannot parse datetimes with
+            #    mixed time zones unless `utc=True`" instead
+            is_same_offsets = len(self.out_tzoffset_vals) == 1
+            if not is_same_offsets or (self.found_naive or self.found_other):
+                # e.g. test_to_datetime_mixed_awareness_mixed_types (array_to_datetime)
+                raise ValueError(
+                    "Mixed timezones detected. Pass utc=True in to_datetime "
+                    "or tz='UTC' in DatetimeIndex to convert to a common timezone."
+                )
+            elif tz_out is not None:
+                # GH#55693
+                tz_offset = self.out_tzoffset_vals.pop()
+                tz_out2 = timezone(timedelta(seconds=tz_offset))
+                if not tz_compare(tz_out, tz_out2):
+                    # e.g. (array_strptime)
+                    #  test_to_datetime_mixed_offsets_with_utc_false_removed
+                    # e.g. test_to_datetime_mixed_tzs_mixed_types (array_to_datetime)
+                    raise ValueError(
+                        "Mixed timezones detected. Pass utc=True in to_datetime "
+                        "or tz='UTC' in DatetimeIndex to convert to a common timezone."
+                    )
+                # e.g. (array_strptime)
+                #  test_guess_datetime_format_with_parseable_formats
+                # e.g. test_to_datetime_mixed_types_matching_tzs (array_to_datetime)
+            else:
+                # e.g. test_to_datetime_iso8601_with_timezone_valid (array_strptime)
+                tz_offset = self.out_tzoffset_vals.pop()
+                tz_out = timezone(timedelta(seconds=tz_offset))
+        elif not utc:
+            if tz_out and (self.found_other or self.found_naive_str):
+                # found_other indicates a tz-naive int, float, dt64, or date
+                # e.g. test_to_datetime_mixed_awareness_mixed_types (array_to_datetime)
+                raise ValueError(
+                    "Mixed timezones detected. Pass utc=True in to_datetime "
+                    "or tz='UTC' in DatetimeIndex to convert to a common timezone."
+                )
+        return tz_out
+
 
 def array_strptime(
     ndarray[object] values,
@@ -319,11 +374,8 @@ def array_strptime(
         npy_datetimestruct dts
         int64_t[::1] iresult
         object val
-        bint seen_datetime_offset = False
         bint is_raise = errors=="raise"
         bint is_coerce = errors=="coerce"
-        bint is_same_offsets
-        set out_tzoffset_vals = set()
         tzinfo tz, tz_out = None
         bint iso_format = format_is_iso(fmt)
         NPY_DATETIMEUNIT out_bestunit, item_reso
@@ -418,15 +470,15 @@ def array_strptime(
                     ) from err
                 if out_local == 1:
                     nsecs = out_tzoffset * 60
-                    out_tzoffset_vals.add(nsecs)
-                    seen_datetime_offset = True
+                    state.out_tzoffset_vals.add(nsecs)
+                    state.found_aware_str = True
                     tz = timezone(timedelta(minutes=out_tzoffset))
                     value = tz_localize_to_utc_single(
                         value, tz, ambiguous="raise", nonexistent=None, creso=creso
                     )
                 else:
                     tz = None
-                    out_tzoffset_vals.add("naive")
+                    state.out_tzoffset_vals.add("naive")
                     state.found_naive_str = True
                 iresult[i] = value
                 continue
@@ -475,12 +527,12 @@ def array_strptime(
                 elif creso == NPY_DATETIMEUNIT.NPY_FR_ms:
                     nsecs = nsecs // 10**3
 
-                out_tzoffset_vals.add(nsecs)
-                seen_datetime_offset = True
+                state.out_tzoffset_vals.add(nsecs)
+                state.found_aware_str = True
             else:
                 state.found_naive_str = True
                 tz = None
-                out_tzoffset_vals.add("naive")
+                state.out_tzoffset_vals.add("naive")
 
         except ValueError as ex:
             ex.args = (
@@ -499,35 +551,7 @@ def array_strptime(
                 raise
             return values, None
 
-    if seen_datetime_offset and not utc:
-        is_same_offsets = len(out_tzoffset_vals) == 1
-        if not is_same_offsets or (state.found_naive or state.found_other):
-            raise ValueError(
-                "Mixed timezones detected. Pass utc=True in to_datetime "
-                "or tz='UTC' in DatetimeIndex to convert to a common timezone."
-            )
-        elif tz_out is not None:
-            # GH#55693
-            tz_offset = out_tzoffset_vals.pop()
-            tz_out2 = timezone(timedelta(seconds=tz_offset))
-            if not tz_compare(tz_out, tz_out2):
-                # e.g. test_to_datetime_mixed_offsets_with_utc_false_removed
-                raise ValueError(
-                    "Mixed timezones detected. Pass utc=True in to_datetime "
-                    "or tz='UTC' in DatetimeIndex to convert to a common timezone."
-                )
-            # e.g. test_guess_datetime_format_with_parseable_formats
-        else:
-            # e.g. test_to_datetime_iso8601_with_timezone_valid
-            tz_offset = out_tzoffset_vals.pop()
-            tz_out = timezone(timedelta(seconds=tz_offset))
-    elif not utc:
-        if tz_out and (state.found_other or state.found_naive_str):
-            # found_other indicates a tz-naive int, float, dt64, or date
-            raise ValueError(
-                "Mixed timezones detected. Pass utc=True in to_datetime "
-                "or tz='UTC' in DatetimeIndex to convert to a common timezone."
-            )
+    tz_out = state.check_for_mixed_inputs(tz_out, utc)
 
     if infer_reso:
         if state.creso_ever_changed:

pandas/core/api.py

@@ -41,7 +41,7 @@
     UInt64Dtype,
 )
 from pandas.core.arrays.string_ import StringDtype
-from pandas.core.construction import array
+from pandas.core.construction import array  # noqa: ICN001
 from pandas.core.flags import Flags
 from pandas.core.groupby import (
     Grouper,

pandas/core/arrays/categorical.py

@@ -2482,7 +2482,6 @@ def unique(self) -> Self:
         ['b', 'a']
         Categories (3, object): ['a' < 'b' < 'c']
         """
-        # pylint: disable=useless-parent-delegation
         return super().unique()
 
     def _cast_quantile_result(self, res_values: np.ndarray) -> np.ndarray:

pandas/core/arrays/datetimelike.py

@@ -506,7 +506,6 @@ def view(self, dtype: Literal["m8[ns]"]) -> TimedeltaArray: ...
     @overload
     def view(self, dtype: Dtype | None = ...) -> ArrayLike: ...
 
-    # pylint: disable-next=useless-parent-delegation
     def view(self, dtype: Dtype | None = None) -> ArrayLike:
         # we need to explicitly call super() method as long as the `@overload`s
         #  are present in this file.

pandas/core/base.py

@@ -127,7 +127,7 @@ def __sizeof__(self) -> int:
         """
         memory_usage = getattr(self, "memory_usage", None)
         if memory_usage:
-            mem = memory_usage(deep=True)  # pylint: disable=not-callable
+            mem = memory_usage(deep=True)
             return int(mem if is_scalar(mem) else mem.sum())
 
         # no memory_usage attribute, so fall back to object's 'sizeof'

pandas/core/computation/ops.py

@@ -115,7 +115,7 @@ def _resolve_name(self):
         res = self.env.resolve(local_name, is_local=is_local)
         self.update(res)
 
-        if hasattr(res, "ndim") and res.ndim > 2:
+        if hasattr(res, "ndim") and isinstance(res.ndim, int) and res.ndim > 2:
             raise NotImplementedError(
                 "N-dimensional objects, where N > 2, are not supported with eval"
             )

pandas/core/dtypes/common.py

@@ -250,7 +250,7 @@ def is_scipy_sparse(arr) -> bool:
     """
     global _is_scipy_sparse
 
-    if _is_scipy_sparse is None:  # pylint: disable=used-before-assignment
+    if _is_scipy_sparse is None:
         try:
             from scipy.sparse import issparse as _is_scipy_sparse
         except ImportError:

pandas/core/generic.py

@@ -6729,7 +6729,7 @@ def _pad_or_backfill(
 
         if axis == 1:
             if not self._mgr.is_single_block and inplace:
-                raise NotImplementedError()
+                raise NotImplementedError
             # e.g. test_align_fill_method
             result = self.T._pad_or_backfill(
                 method=method, limit=limit, limit_area=limit_area

pandas/core/groupby/groupby.py

@@ -5014,7 +5014,7 @@ def shift(
             period = cast(int, period)
             if freq is not None:
                 f = lambda x: x.shift(
-                    period,  # pylint: disable=cell-var-from-loop
+                    period,
                     freq,
                     0,  # axis
                     fill_value,