Merge branch 'main' of https://github.com/pandas-dev/pandas into add_numeric_only_gb

Author: rhshadrach

doc/source/whatsnew/v1.5.0.rst

@@ -499,6 +499,7 @@ Conversion
 - Bug in :meth:`Series.astype` and :meth:`DataFrame.astype` from floating dtype to unsigned integer dtype failing to raise in the presence of negative values (:issue:`45151`)
 - Bug in :func:`array` with ``FloatingDtype`` and values containing float-castable strings incorrectly raising (:issue:`45424`)
 - Bug when comparing string and datetime64ns objects causing ``OverflowError`` exception. (:issue:`45506`)
+- Bug in metaclass of generic abstract dtypes causing :meth:`DataFrame.apply` and :meth:`Series.apply` to raise for the built-in function ``type`` (:issue:`46684`)
 
 Strings
 ^^^^^^^

pandas/_libs/algos.pxd

@@ -1,4 +1,7 @@
-from pandas._libs.dtypes cimport numeric_t
+from pandas._libs.dtypes cimport (
+    numeric_object_t,
+    numeric_t,
+)
 
 
 cdef numeric_t kth_smallest_c(numeric_t* arr, Py_ssize_t k, Py_ssize_t n) nogil
@@ -10,3 +13,10 @@ cdef enum TiebreakEnumType:
     TIEBREAK_FIRST
     TIEBREAK_FIRST_DESCENDING
     TIEBREAK_DENSE
+
+
+cdef numeric_object_t get_rank_nan_fill_val(
+    bint rank_nans_highest,
+    numeric_object_t val,
+    bint is_datetimelike=*,
+)

pandas/_libs/algos.pyx

@@ -822,13 +822,17 @@ def is_monotonic(ndarray[numeric_object_t, ndim=1] arr, bint timelike):
 
 cdef numeric_object_t get_rank_nan_fill_val(
     bint rank_nans_highest,
-    numeric_object_t[:] _=None
+    numeric_object_t val,
+    bint is_datetimelike=False,
 ):
     """
     Return the value we'll use to represent missing values when sorting depending
     on if we'd like missing values to end up at the top/bottom. (The second parameter
     is unused, but needed for fused type specialization)
     """
+    if numeric_object_t is int64_t and is_datetimelike and not rank_nans_highest:
+        return NPY_NAT + 1
+
     if rank_nans_highest:
         if numeric_object_t is object:
             return Infinity()
@@ -854,6 +858,9 @@ cdef numeric_object_t get_rank_nan_fill_val(
         if numeric_object_t is object:
             return NegInfinity()
         elif numeric_object_t is int64_t:
+            # Note(jbrockmendel) 2022-03-15 for reasons unknown, using util.INT64_MIN
+            #  instead of NPY_NAT here causes build warnings and failure in
+            #  test_cummax_i8_at_implementation_bound
             return NPY_NAT
         elif numeric_object_t is int32_t:
             return util.INT32_MIN
@@ -975,7 +982,7 @@ def rank_1d(
     # will flip the ordering to still end up with lowest rank.
     # Symmetric logic applies to `na_option == 'bottom'`
     nans_rank_highest = ascending ^ (na_option == 'top')
-    nan_fill_val = get_rank_nan_fill_val[numeric_object_t](nans_rank_highest)
+    nan_fill_val = get_rank_nan_fill_val(nans_rank_highest, <numeric_object_t>0)
     if nans_rank_highest:
         order = [masked_vals, mask]
     else:
@@ -1335,7 +1342,7 @@ def rank_2d(
 
     nans_rank_highest = ascending ^ (na_option == 'top')
     if check_mask:
-        nan_fill_val = get_rank_nan_fill_val[numeric_object_t](nans_rank_highest)
+        nan_fill_val = get_rank_nan_fill_val(nans_rank_highest, <numeric_object_t>0)
 
         if numeric_object_t is object:
             mask = missing.isnaobj2d(values).view(np.uint8)

pandas/_libs/groupby.pyx

@@ -31,7 +31,10 @@ from numpy.math cimport NAN
 cnp.import_array()
 
 from pandas._libs cimport util
-from pandas._libs.algos cimport kth_smallest_c
+from pandas._libs.algos cimport (
+    get_rank_nan_fill_val,
+    kth_smallest_c,
+)
 
 from pandas._libs.algos import (
     ensure_platform_int,
@@ -989,36 +992,16 @@ cdef inline bint _treat_as_na(numeric_object_t val, bint is_datetimelike) nogil:
         return False
 
 
-cdef numeric_t _get_min_or_max(numeric_t val, bint compute_max, bint is_datetimelike):
+cdef numeric_object_t _get_min_or_max(numeric_object_t val, bint compute_max, bint is_datetimelike):
     """
-    Find either the min or the max supported by numeric_t; 'val' is a placeholder
-    to effectively make numeric_t an argument.
+    Find either the min or the max supported by numeric_object_t; 'val' is a
+    placeholder to effectively make numeric_object_t an argument.
     """
-    if numeric_t is int64_t:
-        if compute_max and is_datetimelike:
-            return -_int64_max
-        # Note(jbrockmendel) 2022-03-15 for reasons unknown, using util.INT64_MIN
-        #  instead of NPY_NAT here causes build warnings and failure in
-        #  test_cummax_i8_at_implementation_bound
-        return NPY_NAT if compute_max else util.INT64_MAX
-    elif numeric_t is int32_t:
-        return util.INT32_MIN if compute_max else util.INT32_MAX
-    elif numeric_t is int16_t:
-        return util.INT16_MIN if compute_max else util.INT16_MAX
-    elif numeric_t is int8_t:
-        return util.INT8_MIN if compute_max else util.INT8_MAX
-
-    elif numeric_t is uint64_t:
-        return 0 if compute_max else util.UINT64_MAX
-    elif numeric_t is uint32_t:
-        return 0 if compute_max else util.UINT32_MAX
-    elif numeric_t is uint16_t:
-        return 0 if compute_max else util.UINT16_MAX
-    elif numeric_t is uint8_t:
-        return 0 if compute_max else util.UINT8_MAX
-
-    else:
-        return -np.inf if compute_max else np.inf
+    return get_rank_nan_fill_val(
+        not compute_max,
+        val=val,
+        is_datetimelike=is_datetimelike,
+    )
 
 
 cdef numeric_t _get_na_val(numeric_t val, bint is_datetimelike):

pandas/_libs/tslibs/conversion.pyx

@@ -2,6 +2,7 @@ import cython
 import numpy as np
 
 cimport numpy as cnp
+from cpython.object cimport PyObject
 from numpy cimport (
     int32_t,
     int64_t,
@@ -273,7 +274,8 @@ def ensure_timedelta64ns(arr: ndarray, copy: bool = True):
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def datetime_to_datetime64(ndarray[object] values):
+def datetime_to_datetime64(ndarray values):
+    # ndarray[object], but can't declare object without ndim
     """
     Convert ndarray of datetime-like objects to int64 array representing
     nanosecond timestamps.
@@ -288,20 +290,27 @@ def datetime_to_datetime64(ndarray[object] values):
     inferred_tz : tzinfo or None
     """
     cdef:
-        Py_ssize_t i, n = len(values)
+        Py_ssize_t i, n = values.size
         object val
-        int64_t[:] iresult
+        int64_t ival
+        ndarray iresult  # int64_t, but can't declare that without specifying ndim
         npy_datetimestruct dts
         _TSObject _ts
         bint found_naive = False
         tzinfo inferred_tz = None
 
-    result = np.empty(n, dtype='M8[ns]')
+        cnp.broadcast mi
+
+    result = np.empty((<object>values).shape, dtype='M8[ns]')
     iresult = result.view('i8')
+
+    mi = cnp.PyArray_MultiIterNew2(iresult, values)
     for i in range(n):
-        val = values[i]
+        # Analogous to: val = values[i]
+        val = <object>(<PyObject**>cnp.PyArray_MultiIter_DATA(mi, 1))[0]
+
         if checknull_with_nat(val):
-            iresult[i] = NPY_NAT
+            ival = NPY_NAT
         elif PyDateTime_Check(val):
             if val.tzinfo is not None:
                 if found_naive:
@@ -314,18 +323,23 @@ def datetime_to_datetime64(ndarray[object] values):
                     inferred_tz = val.tzinfo
 
                 _ts = convert_datetime_to_tsobject(val, None)
-                iresult[i] = _ts.value
+                ival = _ts.value
                 check_dts_bounds(&_ts.dts)
             else:
                 found_naive = True
                 if inferred_tz is not None:
                     raise ValueError('Cannot mix tz-aware with '
                                      'tz-naive values')
-                iresult[i] = pydatetime_to_dt64(val, &dts)
+                ival = pydatetime_to_dt64(val, &dts)
                 check_dts_bounds(&dts)
         else:
             raise TypeError(f'Unrecognized value type: {type(val)}')
 
+        # Analogous to: iresult[i] = ival
+        (<int64_t*>cnp.PyArray_MultiIter_DATA(mi, 0))[0] = ival
+
+        cnp.PyArray_MultiIter_NEXT(mi)
+
     return result, inferred_tz
 
 

pandas/core/arrays/datetimes.py

@@ -2247,10 +2247,9 @@ def objects_to_datetime64ns(
         result = result.reshape(data.shape, order=order)
     except ValueError as err:
         try:
-            values, tz_parsed = conversion.datetime_to_datetime64(data.ravel("K"))
+            values, tz_parsed = conversion.datetime_to_datetime64(data)
             # If tzaware, these values represent unix timestamps, so we
             #  return them as i8 to distinguish from wall times
-            values = values.reshape(data.shape, order=order)
             return values.view("i8"), tz_parsed
         except (ValueError, TypeError):
             raise err

pandas/core/arrays/timedeltas.py

@@ -429,14 +429,15 @@ def _formatter(self, boxed: bool = False):
 
         return get_format_timedelta64(self, box=True)
 
-    @dtl.ravel_compat
     def _format_native_types(
         self, *, na_rep="NaT", date_format=None, **kwargs
     ) -> npt.NDArray[np.object_]:
         from pandas.io.formats.format import get_format_timedelta64
 
         formatter = get_format_timedelta64(self._ndarray, na_rep)
-        return np.array([formatter(x) for x in self._ndarray])
+        # equiv: np.array([formatter(x) for x in self._ndarray])
+        #  but independent of dimension
+        return np.frompyfunc(formatter, 1, 1)(self._ndarray)
 
     # ----------------------------------------------------------------
     # Arithmetic Methods

pandas/core/dtypes/generic.py

@@ -37,14 +37,25 @@
 # define abstract base classes to enable isinstance type checking on our
 # objects
 def create_pandas_abc_type(name, attr, comp):
+    def _check(inst):
+        return getattr(inst, attr, "_typ") in comp
 
     # https://github.com/python/mypy/issues/1006
     # error: 'classmethod' used with a non-method
     @classmethod  # type: ignore[misc]
-    def _check(cls, inst) -> bool:
-        return getattr(inst, attr, "_typ") in comp
+    def _instancecheck(cls, inst) -> bool:
+        return _check(inst) and not isinstance(inst, type)
+
+    @classmethod  # type: ignore[misc]
+    def _subclasscheck(cls, inst) -> bool:
+        # Raise instead of returning False
+        # This is consistent with default __subclasscheck__ behavior
+        if not isinstance(inst, type):
+            raise TypeError("issubclass() arg 1 must be a class")
+
+        return _check(inst)
 
-    dct = {"__instancecheck__": _check, "__subclasscheck__": _check}
+    dct = {"__instancecheck__": _instancecheck, "__subclasscheck__": _subclasscheck}
     meta = type("ABCBase", (type,), dct)
     return meta(name, (), dct)
 

pandas/tests/apply/test_frame_apply.py

@@ -1551,3 +1551,29 @@ def foo(x):
     df = DataFrame({"a": [1, 2, 3]})
     with tm.assert_produces_warning(UserWarning, match="Hello, World!"):
         df.agg([foo])
+
+
+def test_apply_type():
+    # GH 46719
+    df = DataFrame(
+        {"col1": [3, "string", float], "col2": [0.25, datetime(2020, 1, 1), np.nan]},
+        index=["a", "b", "c"],
+    )
+
+    # applymap
+    result = df.applymap(type)
+    expected = DataFrame(
+        {"col1": [int, str, type], "col2": [float, datetime, float]},
+        index=["a", "b", "c"],
+    )
+    tm.assert_frame_equal(result, expected)
+
+    # axis=0
+    result = df.apply(type, axis=0)
+    expected = Series({"col1": Series, "col2": Series})
+    tm.assert_series_equal(result, expected)
+
+    # axis=1
+    result = df.apply(type, axis=1)
+    expected = Series({"a": Series, "b": Series, "c": Series})
+    tm.assert_series_equal(result, expected)

pandas/tests/apply/test_series_apply.py

@@ -889,3 +889,11 @@ def test_apply_retains_column_name():
         index=Index(range(3), name="x"),
     )
     tm.assert_frame_equal(result, expected)
+
+
+def test_apply_type():
+    # GH 46719
+    s = Series([3, "string", float], index=["a", "b", "c"])
+    result = s.apply(type)
+    expected = Series([int, str, type], index=["a", "b", "c"])
+    tm.assert_series_equal(result, expected)

pandas/tests/dtypes/test_generic.py

@@ -1,3 +1,4 @@
+import re
 from warnings import catch_warnings
 
 import numpy as np
@@ -49,13 +50,28 @@ class TestABCClasses:
 
     @pytest.mark.parametrize("abctype1, inst", abc_pairs)
     @pytest.mark.parametrize("abctype2, _", abc_pairs)
-    def test_abc_pairs(self, abctype1, abctype2, inst, _):
-        # GH 38588
+    def test_abc_pairs_instance_check(self, abctype1, abctype2, inst, _):
+        # GH 38588, 46719
         if abctype1 == abctype2:
             assert isinstance(inst, getattr(gt, abctype2))
+            assert not isinstance(type(inst), getattr(gt, abctype2))
         else:
             assert not isinstance(inst, getattr(gt, abctype2))
 
+    @pytest.mark.parametrize("abctype1, inst", abc_pairs)
+    @pytest.mark.parametrize("abctype2, _", abc_pairs)
+    def test_abc_pairs_subclass_check(self, abctype1, abctype2, inst, _):
+        # GH 38588, 46719
+        if abctype1 == abctype2:
+            assert issubclass(type(inst), getattr(gt, abctype2))
+
+            with pytest.raises(
+                TypeError, match=re.escape("issubclass() arg 1 must be a class")
+            ):
+                issubclass(inst, getattr(gt, abctype2))
+        else:
+            assert not issubclass(type(inst), getattr(gt, abctype2))
+
     abc_subclasses = {
         "ABCIndex": [
             abctype