COMPAT: Update code for latest Cython 3 beta (#51640)

* COMPAT: Update code for latest Cython 3 beta

* Update ubuntu.yml

* Update ubuntu.yml

* Update ubuntu.yml

* correctly propagate PANDAS_CI

* Fix more pow

* hack to force Cython to realize correct specialization in join

* use cpow in timedeltas.pyx

* bump cython

* bump cython here as well

* fixes

Author: lithomas1

.github/workflows/ubuntu.yml

@@ -15,9 +15,6 @@ on:
       - "doc/**"
       - "web/**"
 
-env:
-  PANDAS_CI: 1
-
 permissions:
   contents: read
 
@@ -33,6 +30,7 @@ jobs:
         env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml, actions-311.yaml]
         pattern: ["not single_cpu", "single_cpu"]
         pyarrow_version: ["8", "9", "10"]
+        pandas_ci: [1]
         include:
           - name: "Downstream Compat"
             env_file: actions-38-downstream_compat.yaml
@@ -77,6 +75,9 @@ jobs:
             env_file: actions-310-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
+            # TODO(cython3): Re-enable once next-beta(after beta 1) comes out
+            # There are some warnings failing the build with -werror
+            pandas_ci: 0
         exclude:
           - env_file: actions-38.yaml
             pyarrow_version: "8"
@@ -100,6 +101,7 @@ jobs:
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_DATA_MANAGER: ${{ matrix.pandas_data_manager || 'block' }}
       PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
+      PANDAS_CI: ${{ matrix.pandas_ci }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }}
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}

pandas/_libs/algos.pyx

@@ -166,7 +166,7 @@ cpdef ndarray[int64_t, ndim=1] unique_deltas(const int64_t[:] arr):
 
 @cython.wraparound(False)
 @cython.boundscheck(False)
-def is_lexsorted(list_of_arrays: list) -> bint:
+def is_lexsorted(list_of_arrays: list) -> bool:
     cdef:
         Py_ssize_t i
         Py_ssize_t n, nlevels

pandas/_libs/join.pyx

@@ -834,10 +834,10 @@ def asof_join_forward_on_X_by_Y(numeric_t[:] left_values,
     return left_indexer, right_indexer
 
 
-def asof_join_nearest_on_X_by_Y(numeric_t[:] left_values,
-                                numeric_t[:] right_values,
-                                by_t[:] left_by_values,
-                                by_t[:] right_by_values,
+def asof_join_nearest_on_X_by_Y(ndarray[numeric_t] left_values,
+                                ndarray[numeric_t] right_values,
+                                ndarray[by_t] left_by_values,
+                                ndarray[by_t] right_by_values,
                                 bint allow_exact_matches=True,
                                 tolerance=None,
                                 bint use_hashtable=True):
@@ -850,7 +850,17 @@ def asof_join_nearest_on_X_by_Y(numeric_t[:] left_values,
         numeric_t bdiff, fdiff
 
     # search both forward and backward
-    bli, bri = asof_join_backward_on_X_by_Y(
+    # TODO(cython3):
+    # Bug in beta1 preventing Cython from choosing
+    # right specialization when one fused memview is None
+    # Doesn't matter what type we choose
+    # (nothing happens anyways since it is None)
+    # GH 51640
+    if left_by_values is not None and left_by_values.dtype != object:
+        by_dtype = f"{left_by_values.dtype}_t"
+    else:
+        by_dtype = object
+    bli, bri = asof_join_backward_on_X_by_Y[f"{left_values.dtype}_t", by_dtype](
         left_values,
         right_values,
         left_by_values,
@@ -859,7 +869,7 @@ def asof_join_nearest_on_X_by_Y(numeric_t[:] left_values,
         tolerance,
         use_hashtable
     )
-    fli, fri = asof_join_forward_on_X_by_Y(
+    fli, fri = asof_join_forward_on_X_by_Y[f"{left_values.dtype}_t", by_dtype](
         left_values,
         right_values,
         left_by_values,

pandas/_libs/lib.pyx

@@ -150,11 +150,11 @@ def memory_usage_of_objects(arr: object[:]) -> int64_t:
 
     Does not include the actual bytes of the pointers
     """
-    i: Py_ssize_t
-    n: Py_ssize_t
-    size: int64_t
+    cdef:
+        Py_ssize_t i
+        Py_ssize_t n
+        int64_t size = 0
 
-    size = 0
     n = len(arr)
     for i in range(n):
         size += arr[i].__sizeof__()

pandas/_libs/sparse_op_helper.pxi.in

@@ -128,7 +128,9 @@ def get_dispatch(dtypes):
 
 {{for opname, dtype, rdtype in get_dispatch(dtypes)}}
 
-
+{{if opname == "pow"}}
+@cython.cpow(True) # Cython 3 matches Python pow, which isn't what we want here
+{{endif}}
 @cython.wraparound(False)
 @cython.boundscheck(False)
 cdef tuple block_op_{{opname}}_{{dtype}}({{dtype}}_t[:] x_,
@@ -229,7 +231,9 @@ cdef tuple block_op_{{opname}}_{{dtype}}({{dtype}}_t[:] x_,
 
     return out, out_index, {{(opname, 'xfill', 'yfill', dtype) | get_op}}
 
-
+{{if opname == "pow"}}
+@cython.cpow(True) # Cython 3 matches Python pow, which isn't what we want here
+{{endif}}
 @cython.wraparound(False)
 @cython.boundscheck(False)
 cdef tuple int_op_{{opname}}_{{dtype}}({{dtype}}_t[:] x_,

pandas/_libs/tslibs/offsets.pyx

@@ -4564,7 +4564,7 @@ def roll_qtrday(other: datetime, n: int, month: int,
 cdef int _roll_qtrday(npy_datetimestruct* dts,
                       int n,
                       int months_since,
-                      str day_opt) nogil except? -1:
+                      str day_opt) except? -1 nogil:
     """
     See roll_qtrday.__doc__
     """

pandas/_libs/tslibs/timedeltas.pyx

@@ -489,6 +489,7 @@ cdef int64_t _item_to_timedelta64(
             raise
 
 
+@cython.cpow(True)
 cdef int64_t parse_timedelta_string(str ts) except? -1:
     """
     Parse a regular format timedelta string. Return an int64_t (in ns)

pandas/_libs/window/aggregations.pyx

@@ -21,6 +21,8 @@ from numpy cimport (
 
 cnp.import_array()
 
+import cython
+
 from pandas._libs.algos import is_monotonic
 
 
@@ -1733,7 +1735,7 @@ def roll_weighted_var(const float64_t[:] values, const float64_t[:] weights,
 
 # ----------------------------------------------------------------------
 # Exponentially weighted moving
-
+@cython.cpow(True)
 def ewm(const float64_t[:] vals, const int64_t[:] start, const int64_t[:] end,
         int minp, float64_t com, bint adjust, bint ignore_na,
         const float64_t[:] deltas=None, bint normalize=True) -> np.ndarray:

pandas/core/reshape/merge.py

@@ -2160,7 +2160,13 @@ def injection(obj):
         else:
             # choose appropriate function by type
             func = _asof_by_function(self.direction)
-            return func(
+            # TODO(cython3):
+            # Bug in beta1 preventing Cython from choosing
+            # right specialization when one fused memview is None
+            # Doesn't matter what type we choose
+            # (nothing happens anyways since it is None)
+            # GH 51640
+            return func[f"{left_values.dtype}_t", object](
                 left_values,
                 right_values,
                 None,