Merge branch 'main' of https://github.com/pandas-dev/pandas into factorize_na

Author: rhshadrach

doc/source/whatsnew/v1.4.2.rst

@@ -21,7 +21,7 @@ Fixed regressions
 - Fixed regression in :meth:`DataFrame.replace` when a replacement value was also a target for replacement (:issue:`46306`)
 - Fixed regression in :meth:`DataFrame.replace` when the replacement value was explicitly ``None`` when passed in a dictionary to ``to_replace`` (:issue:`45601`, :issue:`45836`)
 - Fixed regression when setting values with :meth:`DataFrame.loc` losing :class:`MultiIndex` names if :class:`DataFrame`  was empty before (:issue:`46317`)
--
+- Fixed regression when rendering boolean datatype columns with :meth:`.Styler` (:issue:`46384`)
 
 .. ---------------------------------------------------------------------------
 
@@ -32,6 +32,7 @@ Bug fixes
 - Fix some cases for subclasses that define their ``_constructor`` properties as general callables (:issue:`46018`)
 - Fixed "longtable" formatting in :meth:`.Styler.to_latex` when ``column_format`` is given in extended format (:issue:`46037`)
 - Fixed incorrect rendering in :meth:`.Styler.format` with ``hyperlinks="html"`` when the url contains a colon or other special characters (:issue:`46389`)
+- Fixed :meth:`Groupby.rolling` with a frequency window that would raise a ``ValueError`` even if the datetimes within each group were monotonic (:issue:`46061`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.5.0.rst

@@ -561,7 +561,8 @@ I/O
 - Bug in :func:`read_parquet` when ``engine="pyarrow"`` which caused partial write to disk when column of unsupported datatype was passed (:issue:`44914`)
 - Bug in :func:`DataFrame.to_excel` and :class:`ExcelWriter` would raise when writing an empty DataFrame to a ``.ods`` file (:issue:`45793`)
 - Bug in Parquet roundtrip for Interval dtype with ``datetime64[ns]`` subtype (:issue:`45881`)
-- Bug in :func:`read_excel` when reading a ``.ods`` file with newlines between xml elements(:issue:`45598`)
+- Bug in :func:`read_excel` when reading a ``.ods`` file with newlines between xml elements (:issue:`45598`)
+- Bug in :func:`read_parquet` when ``engine="fastparquet"`` where the file was not closed on error (:issue:`46555`)
 
 Period
 ^^^^^^

pandas/_libs/tslibs/__init__.py

@@ -27,18 +27,18 @@
 ]
 
 from pandas._libs.tslibs import dtypes
-from pandas._libs.tslibs.conversion import (
-    OutOfBoundsTimedelta,
-    localize_pydatetime,
-)
+from pandas._libs.tslibs.conversion import localize_pydatetime
 from pandas._libs.tslibs.dtypes import Resolution
 from pandas._libs.tslibs.nattype import (
     NaT,
     NaTType,
     iNaT,
     nat_strings,
 )
-from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime
+from pandas._libs.tslibs.np_datetime import (
+    OutOfBoundsDatetime,
+    OutOfBoundsTimedelta,
+)
 from pandas._libs.tslibs.offsets import (
     BaseOffset,
     Tick,

pandas/_libs/tslibs/conversion.pyx

@@ -44,7 +44,10 @@ from pandas._libs.tslibs.np_datetime cimport (
     pydatetime_to_dt64,
 )
 
-from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime
+from pandas._libs.tslibs.np_datetime import (
+    OutOfBoundsDatetime,
+    OutOfBoundsTimedelta,
+)
 
 from pandas._libs.tslibs.timezones cimport (
     get_dst_info,
@@ -85,15 +88,6 @@ DT64NS_DTYPE = np.dtype('M8[ns]')
 TD64NS_DTYPE = np.dtype('m8[ns]')
 
 
-class OutOfBoundsTimedelta(ValueError):
-    """
-    Raised when encountering a timedelta value that cannot be represented
-    as a timedelta64[ns].
-    """
-    # Timedelta analogue to OutOfBoundsDatetime
-    pass
-
-
 # ----------------------------------------------------------------------
 # Unit Conversion Helpers
 

pandas/_libs/tslibs/np_datetime.pxd

@@ -66,6 +66,10 @@ cdef extern from "src/datetime/np_datetime.h":
     npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT fr,
                                                 npy_datetimestruct *d) nogil
 
+    void pandas_timedelta_to_timedeltastruct(npy_timedelta val,
+                                             NPY_DATETIMEUNIT fr,
+                                             pandas_timedeltastruct *result
+                                             ) nogil
 
 cdef bint cmp_scalar(int64_t lhs, int64_t rhs, int op) except -1
 
@@ -94,3 +98,5 @@ cpdef cnp.ndarray astype_overflowsafe(
     cnp.dtype dtype,  # ndarray[datetime64[anyunit]]
     bint copy=*,
 )
+
+cdef bint cmp_dtstructs(npy_datetimestruct* left, npy_datetimestruct* right, int op)

pandas/_libs/tslibs/np_datetime.pyi

@@ -1,9 +1,11 @@
 import numpy as np
 
 class OutOfBoundsDatetime(ValueError): ...
+class OutOfBoundsTimedelta(ValueError): ...
 
 # only exposed for testing
 def py_get_unit_from_dtype(dtype: np.dtype): ...
+def py_td64_to_tdstruct(td64: int, unit: int) -> dict: ...
 def astype_overflowsafe(
     arr: np.ndarray, dtype: np.dtype, copy: bool = ...
 ) -> np.ndarray: ...

pandas/_libs/tslibs/np_datetime.pyx

@@ -34,11 +34,6 @@ cdef extern from "src/datetime/np_datetime.h":
     int cmp_npy_datetimestruct(npy_datetimestruct *a,
                                npy_datetimestruct *b)
 
-    void pandas_timedelta_to_timedeltastruct(npy_timedelta val,
-                                             NPY_DATETIMEUNIT fr,
-                                             pandas_timedeltastruct *result
-                                             ) nogil
-
     # AS, FS, PS versions exist but are not imported because they are not used.
     npy_datetimestruct _NS_MIN_DTS, _NS_MAX_DTS
     npy_datetimestruct _US_MIN_DTS, _US_MAX_DTS
@@ -100,6 +95,28 @@ def py_get_unit_from_dtype(dtype):
 # Comparison
 
 
+cdef bint cmp_dtstructs(
+    npy_datetimestruct* left, npy_datetimestruct* right, int op
+):
+    cdef:
+        int cmp_res
+
+    cmp_res = cmp_npy_datetimestruct(left, right)
+    if op == Py_EQ:
+        return cmp_res == 0
+    if op == Py_NE:
+        return cmp_res != 0
+    if op == Py_GT:
+        return cmp_res == 1
+    if op == Py_LT:
+        return cmp_res == -1
+    if op == Py_GE:
+        return cmp_res == 1 or cmp_res == 0
+    else:
+        # i.e. op == Py_LE
+        return cmp_res == -1 or cmp_res == 0
+
+
 cdef inline bint cmp_scalar(int64_t lhs, int64_t rhs, int op) except -1:
     """
     cmp_scalar is a more performant version of PyObject_RichCompare
@@ -127,6 +144,15 @@ class OutOfBoundsDatetime(ValueError):
     pass
 
 
+class OutOfBoundsTimedelta(ValueError):
+    """
+    Raised when encountering a timedelta value that cannot be represented
+    as a timedelta64[ns].
+    """
+    # Timedelta analogue to OutOfBoundsDatetime
+    pass
+
+
 cdef check_dts_bounds(npy_datetimestruct *dts, NPY_DATETIMEUNIT unit=NPY_FR_ns):
     """Raises OutOfBoundsDatetime if the given date is outside the range that
     can be represented by nanosecond-resolution 64-bit integers."""
@@ -189,6 +215,14 @@ cdef inline void td64_to_tdstruct(int64_t td64,
     return
 
 
+# just exposed for testing at the moment
+def py_td64_to_tdstruct(int64_t td64, NPY_DATETIMEUNIT unit):
+    cdef:
+        pandas_timedeltastruct tds
+    pandas_timedelta_to_timedeltastruct(td64, unit, &tds)
+    return tds  # <- returned as a dict to python
+
+
 cdef inline int64_t pydatetime_to_dt64(datetime val,
                                        npy_datetimestruct *dts):
     """