Merge remote-tracking branch 'upstream/main' into internals-column-setitem

Author: jorisvandenbossche

doc/source/conf.py

@@ -9,13 +9,13 @@
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
-
 from datetime import datetime
 import importlib
 import inspect
 import logging
 import os
 import sys
+import warnings
 
 import jinja2
 from numpydoc.docscrape import NumpyDocString
@@ -640,7 +640,10 @@ def linkcode_resolve(domain, info):
     obj = submod
     for part in fullname.split("."):
         try:
-            obj = getattr(obj, part)
+            with warnings.catch_warnings():
+                # Accessing deprecated objects will generate noisy warnings
+                warnings.simplefilter("ignore", FutureWarning)
+                obj = getattr(obj, part)
         except AttributeError:
             return None
 

doc/source/reference/testing.rst

@@ -26,6 +26,7 @@ Exceptions and warnings
 
    errors.AbstractMethodError
    errors.AccessorRegistrationWarning
+   errors.DataError
    errors.DtypeWarning
    errors.DuplicateLabelError
    errors.EmptyDataError

doc/source/whatsnew/v1.4.3.rst

@@ -18,6 +18,8 @@ Fixed regressions
 - Fixed regression in :func:`read_fwf` raising ``ValueError`` when ``widths`` was specified with ``usecols`` (:issue:`46580`)
 - Fixed regression in :meth:`.Groupby.transform` and :meth:`.Groupby.agg` failing with ``engine="numba"`` when the index was a :class:`MultiIndex` (:issue:`46867`)
 - Fixed regression is :meth:`.Styler.to_latex` and :meth:`.Styler.to_html` where ``buf`` failed in combination with ``encoding`` (:issue:`47053`)
+- Fixed regression in :meth:`.DataFrameGroupBy.agg` when used with list-likes or dict-likes and ``axis=1`` that would give incorrect results; now raises ``NotImplementedError`` (:issue:`46995`)
+- Fixed regression in :meth:`DataFrame.resample` and :meth:`DataFrame.rolling` when used with list-likes or dict-likes and ``axis=1`` that would raise an unintuitive error message; now raises ``NotImplementedError`` (:issue:`46904`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.5.0.rst

@@ -151,6 +151,7 @@ Other enhancements
 - A :class:`errors.PerformanceWarning` is now thrown when using ``string[pyarrow]`` dtype with methods that don't dispatch to ``pyarrow.compute`` methods (:issue:`42613`)
 - Added ``numeric_only`` argument to :meth:`Resampler.sum`, :meth:`Resampler.prod`, :meth:`Resampler.min`, :meth:`Resampler.max`, :meth:`Resampler.first`, and :meth:`Resampler.last` (:issue:`46442`)
 - ``times`` argument in :class:`.ExponentialMovingWindow` now accepts ``np.timedelta64`` (:issue:`47003`)
+- :class:`DataError` now exposed in ``pandas.errors`` (:issue:`27656`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.notable_bug_fixes:
@@ -786,6 +787,8 @@ Groupby/resample/rolling
 - Bug in :meth:`.Rolling.var` would segfault calculating weighted variance when window size was larger than data size (:issue:`46760`)
 - Bug in :meth:`Grouper.__repr__` where ``dropna`` was not included. Now it is (:issue:`46754`)
 - Bug in :meth:`DataFrame.rolling` gives ValueError when center=True, axis=1 and win_type is specified (:issue:`46135`)
+- Bug in :meth:`.DataFrameGroupBy.describe` and :meth:`.SeriesGroupBy.describe` produces inconsistent results for empty datasets (:issue:`41575`)
+-
 
 Reshaping
 ^^^^^^^^^

pandas/_libs/tslib.pyx

@@ -59,6 +59,7 @@ from pandas._libs.tslibs.nattype cimport (
     c_nat_strings as nat_strings,
 )
 from pandas._libs.tslibs.timestamps cimport _Timestamp
+from pandas._libs.tslibs.timezones cimport tz_compare
 
 from pandas._libs.tslibs import (
     Resolution,
@@ -447,6 +448,7 @@ cpdef array_to_datetime(
         bint string_to_dts_failed
         datetime py_dt
         tzinfo tz_out = None
+        bint found_tz = False, found_naive = False
 
     # specify error conditions
     assert is_raise or is_ignore or is_coerce
@@ -465,18 +467,34 @@ cpdef array_to_datetime(
                 elif PyDateTime_Check(val):
                     seen_datetime = True
                     if val.tzinfo is not None:
+                        found_tz = True
                         if utc_convert:
                             _ts = convert_datetime_to_tsobject(val, None)
                             iresult[i] = _ts.value
-                        else:
+                        elif found_naive:
                             raise ValueError('Tz-aware datetime.datetime '
                                              'cannot be converted to '
                                              'datetime64 unless utc=True')
-                    elif isinstance(val, _Timestamp):
-                        iresult[i] = val.value
+                        elif tz_out is not None and not tz_compare(tz_out, val.tzinfo):
+                            raise ValueError('Tz-aware datetime.datetime '
+                                             'cannot be converted to '
+                                             'datetime64 unless utc=True')
+                        else:
+                            found_tz = True
+                            tz_out = val.tzinfo
+                            _ts = convert_datetime_to_tsobject(val, None)
+                            iresult[i] = _ts.value
+
                     else:
-                        iresult[i] = pydatetime_to_dt64(val, &dts)
-                        check_dts_bounds(&dts)
+                        found_naive = True
+                        if found_tz:
+                            raise ValueError('Cannot mix tz-aware with '
+                                             'tz-naive values')
+                        if isinstance(val, _Timestamp):
+                            iresult[i] = val.value
+                        else:
+                            iresult[i] = pydatetime_to_dt64(val, &dts)
+                            check_dts_bounds(&dts)
 
                 elif PyDate_Check(val):
                     seen_datetime = True

pandas/_libs/tslibs/conversion.pyi

@@ -23,7 +23,4 @@ def ensure_timedelta64ns(
     arr: np.ndarray,  # np.ndarray[timedelta64[ANY]]
     copy: bool = ...,
 ) -> np.ndarray: ...  # np.ndarray[timedelta64ns]
-def datetime_to_datetime64(
-    values: npt.NDArray[np.object_],
-) -> tuple[np.ndarray, tzinfo | None]: ...  # (np.ndarray[dt64ns], _)
 def localize_pydatetime(dt: datetime, tz: tzinfo | None) -> datetime: ...

pandas/_libs/tslibs/conversion.pyx

@@ -264,80 +264,6 @@ def ensure_timedelta64ns(arr: ndarray, copy: bool = True):
     return dt64_result.view(TD64NS_DTYPE)
 
 
-# ----------------------------------------------------------------------
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-def datetime_to_datetime64(ndarray values):
-    # ndarray[object], but can't declare object without ndim
-    """
-    Convert ndarray of datetime-like objects to int64 array representing
-    nanosecond timestamps.
-
-    Parameters
-    ----------
-    values : ndarray[object]
-
-    Returns
-    -------
-    result : ndarray[datetime64ns]
-    inferred_tz : tzinfo or None
-    """
-    cdef:
-        Py_ssize_t i, n = values.size
-        object val
-        int64_t ival
-        ndarray iresult  # int64_t, but can't declare that without specifying ndim
-        npy_datetimestruct dts
-        _TSObject _ts
-        bint found_naive = False
-        tzinfo inferred_tz = None
-
-        cnp.broadcast mi
-
-    result = np.empty((<object>values).shape, dtype='M8[ns]')
-    iresult = result.view('i8')
-
-    mi = cnp.PyArray_MultiIterNew2(iresult, values)
-    for i in range(n):
-        # Analogous to: val = values[i]
-        val = <object>(<PyObject**>cnp.PyArray_MultiIter_DATA(mi, 1))[0]
-
-        if checknull_with_nat(val):
-            ival = NPY_NAT
-        elif PyDateTime_Check(val):
-            if val.tzinfo is not None:
-                if found_naive:
-                    raise ValueError('Cannot mix tz-aware with '
-                                     'tz-naive values')
-                if inferred_tz is not None:
-                    if not tz_compare(val.tzinfo, inferred_tz):
-                        raise ValueError('Array must be all same time zone')
-                else:
-                    inferred_tz = val.tzinfo
-
-                _ts = convert_datetime_to_tsobject(val, None)
-                ival = _ts.value
-                check_dts_bounds(&_ts.dts)
-            else:
-                found_naive = True
-                if inferred_tz is not None:
-                    raise ValueError('Cannot mix tz-aware with '
-                                     'tz-naive values')
-                ival = pydatetime_to_dt64(val, &dts)
-                check_dts_bounds(&dts)
-        else:
-            raise TypeError(f'Unrecognized value type: {type(val)}')
-
-        # Analogous to: iresult[i] = ival
-        (<int64_t*>cnp.PyArray_MultiIter_DATA(mi, 0))[0] = ival
-
-        cnp.PyArray_MultiIter_NEXT(mi)
-
-    return result, inferred_tz
-
-
 # ----------------------------------------------------------------------
 # _TSObject Conversion
 

pandas/conftest.py

@@ -105,6 +105,24 @@ def pytest_addoption(parser):
     )
 
 
+def ignore_doctest_warning(item: pytest.Item, path: str, message: str) -> None:
+    """Ignore doctest warning.
+
+    Parameters
+    ----------
+    item : pytest.Item
+        pytest test item.
+    path : str
+        Module path to Python object, e.g. "pandas.core.frame.DataFrame.append". A
+        warning will be filtered when item.name ends with in given path. So it is
+        sufficient to specify e.g. "DataFrame.append".
+    message : str
+        Message to be filtered.
+    """
+    if item.name.endswith(path):
+        item.add_marker(pytest.mark.filterwarnings(f"ignore:{message}"))
+
+
 def pytest_collection_modifyitems(items, config):
     skip_slow = config.getoption("--skip-slow")
     only_slow = config.getoption("--only-slow")
@@ -117,13 +135,34 @@ def pytest_collection_modifyitems(items, config):
         (pytest.mark.db, "db", skip_db, "--skip-db"),
     ]
 
+    # Warnings from doctests that can be ignored; place reason in comment above.
+    # Each entry specifies (path, message) - see the ignore_doctest_warning function
+    ignored_doctest_warnings = [
+        # Deprecations where the docstring will emit a warning
+        ("DataFrame.append", "The frame.append method is deprecated"),
+        ("Series.append", "The series.append method is deprecated"),
+        ("dtypes.common.is_categorical", "is_categorical is deprecated"),
+        ("Categorical.replace", "Categorical.replace is deprecated"),
+        ("dtypes.common.is_extension_type", "'is_extension_type' is deprecated"),
+        ("Index.is_mixed", "Index.is_mixed is deprecated"),
+        ("MultiIndex._is_lexsorted", "MultiIndex.is_lexsorted is deprecated"),
+        # Docstring divides by zero to show behavior difference
+        ("missing.mask_zero_div_zero", "divide by zero encountered"),
+        # Docstring demonstrates the call raises a warning
+        ("_validators.validate_axis_style_args", "Use named arguments"),
+    ]
+
     for item in items:
         if config.getoption("--doctest-modules") or config.getoption(
             "--doctest-cython", default=False
         ):
             # autouse=True for the add_doctest_imports can lead to expensive teardowns
             # since doctest_namespace is a session fixture
             item.add_marker(pytest.mark.usefixtures("add_doctest_imports"))
+
+            for path, message in ignored_doctest_warnings:
+                ignore_doctest_warning(item, path, message)
+
         # mark all tests in the pandas/tests/frame directory with "arraymanager"
         if "/frame/" in item.nodeid:
             item.add_marker(pytest.mark.arraymanager)

pandas/core/apply.py

@@ -33,6 +33,7 @@
     Axis,
     NDFrameT,
 )
+from pandas.errors import DataError
 from pandas.util._decorators import cache_readonly
 from pandas.util._exceptions import find_stack_level
 
@@ -51,7 +52,6 @@
 
 from pandas.core.algorithms import safe_sort
 from pandas.core.base import (
-    DataError,
     SelectionMixin,
     SpecificationError,
 )
@@ -325,6 +325,9 @@ def agg_list_like(self) -> DataFrame | Series:
         obj = self.obj
         arg = cast(List[AggFuncTypeBase], self.f)
 
+        if getattr(obj, "axis", 0) == 1:
+            raise NotImplementedError("axis other than 0 is not supported")
+
         if not isinstance(obj, SelectionMixin):
             # i.e. obj is Series or DataFrame
             selected_obj = obj
@@ -456,6 +459,9 @@ def agg_dict_like(self) -> DataFrame | Series:
         obj = self.obj
         arg = cast(AggFuncTypeDict, self.f)
 
+        if getattr(obj, "axis", 0) == 1:
+            raise NotImplementedError("axis other than 0 is not supported")
+
         if not isinstance(obj, SelectionMixin):
             # i.e. obj is Series or DataFrame
             selected_obj = obj

pandas/core/arrays/datetimes.py

@@ -2263,14 +2263,6 @@ def objects_to_datetime64ns(
             allow_mixed=allow_mixed,
         )
         result = result.reshape(data.shape, order=order)
-    except ValueError as err:
-        try:
-            values, tz_parsed = conversion.datetime_to_datetime64(data)
-            # If tzaware, these values represent unix timestamps, so we
-            #  return them as i8 to distinguish from wall times
-            return values.view("i8"), tz_parsed
-        except (ValueError, TypeError):
-            raise err
     except OverflowError as err:
         # Exception is raised when a part of date is greater than 32 bit signed int
         raise OutOfBoundsDatetime("Out of bounds nanosecond timestamp") from err

pandas/core/arrays/sparse/array.py

@@ -1253,7 +1253,7 @@ def astype(self, dtype: AstypeArg | None = None, copy: bool = True):
         IntIndex
         Indices: array([2, 3], dtype=int32)
 
-        >>> arr.astype(np.dtype('int32'))
+        >>> arr.astype(SparseDtype(np.dtype('int32')))
         [0, 0, 1, 2]
         Fill: 0
         IntIndex
@@ -1262,19 +1262,19 @@ def astype(self, dtype: AstypeArg | None = None, copy: bool = True):
         Using a NumPy dtype with a different kind (e.g. float) will coerce
         just ``self.sp_values``.
 
-        >>> arr.astype(np.dtype('float64'))
+        >>> arr.astype(SparseDtype(np.dtype('float64')))
         ... # doctest: +NORMALIZE_WHITESPACE
-        [0.0, 0.0, 1.0, 2.0]
-        Fill: 0.0
+        [nan, nan, 1.0, 2.0]
+        Fill: nan
         IntIndex
         Indices: array([2, 3], dtype=int32)
 
-        Use a SparseDtype if you wish to be change the fill value as well.
+        Using a SparseDtype, you can also change the fill value as well.
 
-        >>> arr.astype(SparseDtype("float64", fill_value=np.nan))
+        >>> arr.astype(SparseDtype("float64", fill_value=0.0))
         ... # doctest: +NORMALIZE_WHITESPACE
-        [nan, nan, 1.0, 2.0]
-        Fill: nan
+        [0.0, 0.0, 1.0, 2.0]
+        Fill: 0.0
         IntIndex
         Indices: array([2, 3], dtype=int32)
         """

pandas/core/base.py

@@ -176,10 +176,6 @@ def __setattr__(self, key: str, value):
         object.__setattr__(self, key, value)
 
 
-class DataError(Exception):
-    pass
-
-
 class SpecificationError(Exception):
     pass