Merge branch 'main' into sas/byteswap

Author: jonashaag

.github/workflows/macos-windows.yml

@@ -23,7 +23,7 @@ jobs:
     defaults:
       run:
         shell: bash -el {0}
-    timeout-minutes: 90
+    timeout-minutes: 120
     strategy:
       matrix:
         os: [macos-latest, windows-latest]

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
         pass_filenames: true
         require_serial: false
 -   repo: https://github.com/python/black
-    rev: 22.3.0
+    rev: 22.6.0
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
@@ -27,7 +27,7 @@ repos:
     -   id: codespell
         types_or: [python, rst, markdown]
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.2.0
+    rev: v4.3.0
     hooks:
     -   id: debug-statements
     -   id: end-of-file-fixer
@@ -59,7 +59,7 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.32.1
+    rev: v2.34.0
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -74,7 +74,7 @@ repos:
         types: [text]  # overwrite types: [rst]
         types_or: [python, rst]
 -   repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.6
+    rev: v0.6.1
     hooks:
     - id: sphinx-lint
 -   repo: https://github.com/asottile/yesqa
@@ -229,3 +229,11 @@ repos:
         entry: python scripts/validate_min_versions_in_sync.py
         language: python
         files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$
+    -   id: flake8-pyi
+        name: flake8-pyi
+        entry: flake8 --extend-ignore=E301,E302,E305,E701,E704
+        types: [pyi]
+        language: python
+        additional_dependencies:
+        - flake8==4.0.1
+        - flake8-pyi==22.5.1

RELEASE.md

@@ -1,6 +1,6 @@
 Release Notes
 =============
 
-The list of changes to Pandas between each release can be found
+The list of changes to pandas between each release can be found
 [here](https://pandas.pydata.org/pandas-docs/stable/whatsnew/index.html). For full
 details, see the commit logs at https://github.com/pandas-dev/pandas.

doc/redirects.csv

@@ -761,6 +761,7 @@ generated/pandas.IntervalIndex.mid,../reference/api/pandas.IntervalIndex.mid
 generated/pandas.IntervalIndex.overlaps,../reference/api/pandas.IntervalIndex.overlaps
 generated/pandas.IntervalIndex.right,../reference/api/pandas.IntervalIndex.right
 generated/pandas.IntervalIndex.set_closed,../reference/api/pandas.IntervalIndex.set_closed
+generated/pandas.IntervalIndex.set_inclusive,../reference/api/pandas.IntervalIndex.set_inclusive
 generated/pandas.IntervalIndex.to_tuples,../reference/api/pandas.IntervalIndex.to_tuples
 generated/pandas.IntervalIndex.values,../reference/api/pandas.IntervalIndex.values
 generated/pandas.Interval.left,../reference/api/pandas.Interval.left

doc/source/reference/arrays.rst

@@ -304,6 +304,7 @@ Properties
    :toctree: api/
 
    Interval.inclusive
+   Interval.closed
    Interval.closed_left
    Interval.closed_right
    Interval.is_empty
@@ -351,6 +352,7 @@ A collection of intervals may be stored in an :class:`arrays.IntervalArray`.
       arrays.IntervalArray.contains
       arrays.IntervalArray.overlaps
       arrays.IntervalArray.set_closed
+      arrays.IntervalArray.set_inclusive
       arrays.IntervalArray.to_tuples
 
 

doc/source/reference/indexing.rst

@@ -251,6 +251,7 @@ IntervalIndex components
    IntervalIndex.get_loc
    IntervalIndex.get_indexer
    IntervalIndex.set_closed
+   IntervalIndex.set_inclusive
    IntervalIndex.contains
    IntervalIndex.overlaps
    IntervalIndex.to_tuples

doc/source/reference/testing.rst

@@ -26,6 +26,7 @@ Exceptions and warnings
 
    errors.AbstractMethodError
    errors.AccessorRegistrationWarning
+   errors.CSSWarning
    errors.DataError
    errors.DtypeWarning
    errors.DuplicateLabelError

doc/source/user_guide/groupby.rst

@@ -839,10 +839,10 @@ Alternatively, the built-in methods could be used to produce the same outputs.
 
 .. ipython:: python
 
-   max = ts.groupby(lambda x: x.year).transform("max")
-   min = ts.groupby(lambda x: x.year).transform("min")
+   max_ts = ts.groupby(lambda x: x.year).transform("max")
+   min_ts = ts.groupby(lambda x: x.year).transform("min")
 
-   max - min
+   max_ts - min_ts
 
 Another common data transform is to replace missing data with the group mean.
 

doc/source/whatsnew/v1.4.4.rst

@@ -24,7 +24,7 @@ Fixed regressions
 Bug fixes
 ~~~~~~~~~
 - The :class:`errors.FutureWarning` raised when passing arguments (other than ``filepath_or_buffer``) as positional in :func:`read_csv` is now raised at the correct stacklevel (:issue:`47385`)
--
+- Bug in :meth:`DataFrame.to_sql` when ``method`` was a ``callable`` that did not return an ``int`` and would raise a ``TypeError`` (:issue:`46891`)
 
 .. ---------------------------------------------------------------------------
 

doc/source/whatsnew/v1.5.0.rst

@@ -275,6 +275,9 @@ Other enhancements
 - :class:`.DataError`, :class:`.SpecificationError`, :class:`.SettingWithCopyError`, :class:`.SettingWithCopyWarning`, :class:`.NumExprClobberingError`, :class:`.UndefinedVariableError`, and :class:`.IndexingError` are now exposed in ``pandas.errors`` (:issue:`27656`)
 - Added ``check_like`` argument to :func:`testing.assert_series_equal` (:issue:`47247`)
 - Allow reading compressed SAS files with :func:`read_sas` (e.g., ``.sas7bdat.gz`` files)
+- :meth:`DatetimeIndex.astype` now supports casting timezone-naive indexes to ``datetime64[s]``, ``datetime64[ms]``, and ``datetime64[us]``, and timezone-aware indexes to the corresponding ``datetime64[unit, tzname]`` dtypes (:issue:`47579`)
+- :class:`Series` reducers (e.g. ``min``, ``max``, ``sum``, ``mean``) will now successfully operate when the dtype is numeric and ``numeric_only=True`` is provided; previously this would raise a ``NotImplementedError`` (:issue:`47500`)
+- :meth:`RangeIndex.union` now can return a :class:`RangeIndex` instead of a :class:`Int64Index` if the resulting values are equally spaced (:issue:`47557`, :issue:`43885`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.notable_bug_fixes:
@@ -759,14 +762,15 @@ Other Deprecations
 - Deprecated the ``closed`` argument in :class:`IntervalIndex` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated the ``closed`` argument in :class:`IntervalDtype` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated the ``closed`` argument in :class:`.IntervalArray` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
-- Deprecated the ``closed`` argument in :class:`IntervalTree` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
+- Deprecated :meth:`.IntervalArray.set_closed` and :meth:`.IntervalIndex.set_closed` in favor of ``set_inclusive``; In a future version ``set_closed`` will get removed (:issue:`40245`)
 - Deprecated the ``closed`` argument in :class:`ArrowInterval` in favor of ``inclusive`` argument; In a future version passing ``closed`` will raise (:issue:`40245`)
 - Deprecated allowing ``unit="M"`` or ``unit="Y"`` in :class:`Timestamp` constructor with a non-round float value (:issue:`47267`)
 - Deprecated the ``display.column_space`` global configuration option (:issue:`7576`)
 - Deprecated the argument ``na_sentinel`` in :func:`factorize`, :meth:`Index.factorize`, and :meth:`.ExtensionArray.factorize`; pass ``use_na_sentinel=True`` instead to use the sentinel ``-1`` for NaN values and ``use_na_sentinel=False`` instead of ``na_sentinel=None`` to encode NaN values (:issue:`46910`)
 - Deprecated :meth:`DataFrameGroupBy.transform` not aligning the result when the UDF returned DataFrame (:issue:`45648`)
 - Clarified warning from :func:`to_datetime` when delimited dates can't be parsed in accordance to specified ``dayfirst`` argument (:issue:`46210`)
-
+- Deprecated :class:`Series` and :class:`Resampler` reducers (e.g. ``min``, ``max``, ``sum``, ``mean``) raising a ``NotImplementedError`` when the dtype is non-numric and ``numeric_only=True`` is provided; this will raise a ``TypeError`` in a future version (:issue:`47500`)
+- Deprecated :meth:`Series.rank` returning an empty result when the dtype is non-numeric and ``numeric_only=True`` is provided; this will raise a ``TypeError`` in a future version (:issue:`47500`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_150.performance:
@@ -852,6 +856,7 @@ Conversion
 - Bug in metaclass of generic abstract dtypes causing :meth:`DataFrame.apply` and :meth:`Series.apply` to raise for the built-in function ``type`` (:issue:`46684`)
 - Bug in :meth:`DataFrame.to_records` returning inconsistent numpy types if the index was a :class:`MultiIndex` (:issue:`47263`)
 - Bug in :meth:`DataFrame.to_dict` for ``orient="list"`` or ``orient="index"`` was not returning native types (:issue:`46751`)
+- Bug in :meth:`DataFrame.apply` that returns a :class:`DataFrame` instead of a :class:`Series` when applied to an empty :class:`DataFrame` and ``axis=1`` (:issue:`39111`)
 
 Strings
 ^^^^^^^
@@ -880,8 +885,10 @@ Indexing
 - Bug when setting a value too large for a :class:`Series` dtype failing to coerce to a common type (:issue:`26049`, :issue:`32878`)
 - Bug in :meth:`loc.__setitem__` treating ``range`` keys as positional instead of label-based (:issue:`45479`)
 - Bug in :meth:`Series.__setitem__` when setting ``boolean`` dtype values containing ``NA`` incorrectly raising instead of casting to ``boolean`` dtype (:issue:`45462`)
+- Bug in :meth:`Series.loc` raising with boolean indexer containing ``NA`` when :class:`Index` did not match (:issue:`46551`)
 - Bug in :meth:`Series.__setitem__` where setting :attr:`NA` into a numeric-dtype :class:`Series` would incorrectly upcast to object-dtype rather than treating the value as ``np.nan`` (:issue:`44199`)
 - Bug in :meth:`DataFrame.loc` when setting values to a column and right hand side is a dictionary (:issue:`47216`)
+- Bug in :meth:`DataFrame.loc` when setting a :class:`DataFrame` not aligning index in some cases (:issue:`47578`)
 - Bug in :meth:`Series.__setitem__` with ``datetime64[ns]`` dtype, an all-``False`` boolean mask, and an incompatible value incorrectly casting to ``object`` instead of retaining ``datetime64[ns]`` dtype (:issue:`45967`)
 - Bug in :meth:`Index.__getitem__`  raising ``ValueError`` when indexer is from boolean dtype with ``NA`` (:issue:`45806`)
 - Bug in :meth:`Series.__setitem__` losing precision when enlarging :class:`Series` with scalar (:issue:`32346`)
@@ -934,13 +941,15 @@ I/O
 - Bug in :func:`read_parquet` when ``engine="fastparquet"`` where the file was not closed on error (:issue:`46555`)
 - :meth:`to_html` now excludes the ``border`` attribute from ``<table>`` elements when ``border`` keyword is set to ``False``.
 - Bug in :func:`read_sas` with certain types of compressed SAS7BDAT files (:issue:`35545`)
+- Bug in :func:`read_excel` not forward filling :class:`MultiIndex` when no names were given (:issue:`47487`)
 - Bug in :func:`read_sas` returned ``None`` rather than an empty DataFrame for SAS7BDAT files with zero rows (:issue:`18198`)
 - Bug in :class:`StataWriter` where value labels were always written with default encoding (:issue:`46750`)
 - Bug in :class:`StataWriterUTF8` where some valid characters were removed from variable names (:issue:`47276`)
 - Bug in :meth:`DataFrame.to_excel` when writing an empty dataframe with :class:`MultiIndex` (:issue:`19543`)
 - Bug in :func:`read_sas` with RLE-compressed SAS7BDAT files that contain 0x40 control bytes (:issue:`31243`)
 - Bug in :func:`read_sas` that scrambled column names (:issue:`31243`)
--
+- Bug in :func:`read_sas` with RLE-compressed SAS7BDAT files that contain 0x00 control bytes (:issue:`47099`)
+- Bug in :func:`read_parquet` with ``use_nullable_dtypes=True`` where ``float64`` dtype was returned instead of nullable ``Float64`` dtype (:issue:`45694`)
 
 Period
 ^^^^^^
@@ -995,11 +1004,12 @@ Reshaping
 - Bug in :func:`get_dummies` that selected object and categorical dtypes but not string (:issue:`44965`)
 - Bug in :meth:`DataFrame.align` when aligning a :class:`MultiIndex` to a :class:`Series` with another :class:`MultiIndex` (:issue:`46001`)
 - Bug in concatenation with ``IntegerDtype``, or ``FloatingDtype`` arrays where the resulting dtype did not mirror the behavior of the non-nullable dtypes (:issue:`46379`)
+- Bug in :func:`concat` losing dtype of columns when ``join="outer"`` and ``sort=True`` (:issue:`47329`)
 - Bug in :func:`concat` not sorting the column names when ``None`` is included (:issue:`47331`)
 - Bug in :func:`concat` with identical key leads to error when indexing :class:`MultiIndex` (:issue:`46519`)
 - Bug in :meth:`DataFrame.join` with a list when using suffixes to join DataFrames with duplicate column names (:issue:`46396`)
 - Bug in :meth:`DataFrame.pivot_table` with ``sort=False`` results in sorted index (:issue:`17041`)
--
+- Bug in :meth:`concat` when ``axis=1`` and ``sort=False`` where the resulting Index was a :class:`Int64Index` instead of a :class:`RangeIndex` (:issue:`46675`)
 
 Sparse
 ^^^^^^

pandas/__init__.py

@@ -1,4 +1,5 @@
 # flake8: noqa
+from __future__ import annotations
 
 __docformat__ = "restructuredtext"
 
@@ -185,7 +186,7 @@
 __deprecated_num_index_names = ["Float64Index", "Int64Index", "UInt64Index"]
 
 
-def __dir__():
+def __dir__() -> list[str]:
     # GH43028
     # Int64Index etc. are deprecated, but we still want them to be available in the dir.
     # Remove in Pandas 2.0, when we remove Int64Index etc. from the code base.

pandas/_config/config.py

@@ -60,6 +60,7 @@
     Callable,
     Generic,
     Iterable,
+    Iterator,
     NamedTuple,
     cast,
 )
@@ -435,13 +436,13 @@ def __init__(self, *args) -> None:
 
         self.ops = list(zip(args[::2], args[1::2]))
 
-    def __enter__(self):
+    def __enter__(self) -> None:
         self.undo = [(pat, _get_option(pat, silent=True)) for pat, val in self.ops]
 
         for pat, val in self.ops:
             _set_option(pat, val, silent=True)
 
-    def __exit__(self, *args):
+    def __exit__(self, *args) -> None:
         if self.undo:
             for pat, val in self.undo:
                 _set_option(pat, val, silent=True)
@@ -733,7 +734,7 @@ def pp(name: str, ks: Iterable[str]) -> list[str]:
 
 
 @contextmanager
-def config_prefix(prefix):
+def config_prefix(prefix) -> Iterator[None]:
     """
     contextmanager for multiple invocations of API with a common prefix
 

pandas/_config/localization.py

@@ -39,7 +39,8 @@ def set_locale(
     particular locale, without globally setting the locale. This probably isn't
     thread-safe.
     """
-    current_locale = locale.getlocale()
+    # getlocale is not always compliant with setlocale, use setlocale. GH#46595
+    current_locale = locale.setlocale(lc_var)
 
     try:
         locale.setlocale(lc_var, new_locale)

pandas/_libs/algos.pyx

@@ -324,6 +324,7 @@ def kth_smallest(numeric_t[::1] arr, Py_ssize_t k) -> numeric_t:
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
+@cython.cdivision(True)
 def nancorr(const float64_t[:, :] mat, bint cov=False, minp=None):
     cdef:
         Py_ssize_t i, j, xi, yi, N, K
@@ -356,8 +357,8 @@ def nancorr(const float64_t[:, :] mat, bint cov=False, minp=None):
                         nobs += 1
                         dx = vx - meanx
                         dy = vy - meany
-                        meanx += 1 / nobs * dx
-                        meany += 1 / nobs * dy
+                        meanx += 1. / nobs * dx
+                        meany += 1. / nobs * dy
                         ssqdmx += (vx - meanx) * dx
                         ssqdmy += (vy - meany) * dy
                         covxy += (vx - meanx) * dy

pandas/_libs/index.pyi

@@ -69,7 +69,7 @@ class BaseMultiIndexCodesEngine:
     ) -> npt.NDArray[np.intp]: ...
 
 class ExtensionEngine:
-    def __init__(self, values: "ExtensionArray"): ...
+    def __init__(self, values: ExtensionArray): ...
     def __contains__(self, val: object) -> bool: ...
     def get_loc(self, val: object) -> int | slice | np.ndarray: ...
     def get_indexer(self, values: np.ndarray) -> npt.NDArray[np.intp]: ...

pandas/_libs/interval.pyi

@@ -4,7 +4,6 @@ from typing import (
     Any,
     Generic,
     TypeVar,
-    Union,
     overload,
 )
 
@@ -66,6 +65,8 @@ class Interval(IntervalMixin, Generic[_OrderableT]):
     def right(self: Interval[_OrderableT]) -> _OrderableT: ...
     @property
     def inclusive(self) -> IntervalClosedType: ...
+    @property
+    def closed(self) -> IntervalClosedType: ...
     mid: _MidDescriptor
     length: _LengthDescriptor
     def __init__(
@@ -81,11 +82,7 @@ class Interval(IntervalMixin, Generic[_OrderableT]):
         self: Interval[_OrderableTimesT], key: _OrderableTimesT
     ) -> bool: ...
     @overload
-    def __contains__(
-        self: Interval[_OrderableScalarT], key: Union[int, float]
-    ) -> bool: ...
-    def __repr__(self) -> str: ...
-    def __str__(self) -> str: ...
+    def __contains__(self: Interval[_OrderableScalarT], key: int | float) -> bool: ...
     @overload
     def __add__(
         self: Interval[_OrderableTimesT], y: Timedelta
@@ -95,7 +92,7 @@ class Interval(IntervalMixin, Generic[_OrderableT]):
         self: Interval[int], y: _OrderableScalarT
     ) -> Interval[_OrderableScalarT]: ...
     @overload
-    def __add__(self: Interval[float], y: Union[int, float]) -> Interval[float]: ...
+    def __add__(self: Interval[float], y: int | float) -> Interval[float]: ...
     @overload
     def __radd__(
         self: Interval[_OrderableTimesT], y: Timedelta
@@ -105,7 +102,7 @@ class Interval(IntervalMixin, Generic[_OrderableT]):
         self: Interval[int], y: _OrderableScalarT
     ) -> Interval[_OrderableScalarT]: ...
     @overload
-    def __radd__(self: Interval[float], y: Union[int, float]) -> Interval[float]: ...
+    def __radd__(self: Interval[float], y: int | float) -> Interval[float]: ...
     @overload
     def __sub__(
         self: Interval[_OrderableTimesT], y: Timedelta
@@ -115,7 +112,7 @@ class Interval(IntervalMixin, Generic[_OrderableT]):
         self: Interval[int], y: _OrderableScalarT
     ) -> Interval[_OrderableScalarT]: ...
     @overload
-    def __sub__(self: Interval[float], y: Union[int, float]) -> Interval[float]: ...
+    def __sub__(self: Interval[float], y: int | float) -> Interval[float]: ...
     @overload
     def __rsub__(
         self: Interval[_OrderableTimesT], y: Timedelta
@@ -125,33 +122,31 @@ class Interval(IntervalMixin, Generic[_OrderableT]):
         self: Interval[int], y: _OrderableScalarT
     ) -> Interval[_OrderableScalarT]: ...
     @overload
-    def __rsub__(self: Interval[float], y: Union[int, float]) -> Interval[float]: ...
+    def __rsub__(self: Interval[float], y: int | float) -> Interval[float]: ...
     @overload
     def __mul__(
         self: Interval[int], y: _OrderableScalarT
     ) -> Interval[_OrderableScalarT]: ...
     @overload
-    def __mul__(self: Interval[float], y: Union[int, float]) -> Interval[float]: ...
+    def __mul__(self: Interval[float], y: int | float) -> Interval[float]: ...
     @overload
     def __rmul__(
         self: Interval[int], y: _OrderableScalarT
     ) -> Interval[_OrderableScalarT]: ...
     @overload
-    def __rmul__(self: Interval[float], y: Union[int, float]) -> Interval[float]: ...
+    def __rmul__(self: Interval[float], y: int | float) -> Interval[float]: ...
     @overload
     def __truediv__(
         self: Interval[int], y: _OrderableScalarT
     ) -> Interval[_OrderableScalarT]: ...
     @overload
-    def __truediv__(self: Interval[float], y: Union[int, float]) -> Interval[float]: ...
+    def __truediv__(self: Interval[float], y: int | float) -> Interval[float]: ...
     @overload
     def __floordiv__(
         self: Interval[int], y: _OrderableScalarT
     ) -> Interval[_OrderableScalarT]: ...
     @overload
-    def __floordiv__(
-        self: Interval[float], y: Union[int, float]
-    ) -> Interval[float]: ...
+    def __floordiv__(self: Interval[float], y: int | float) -> Interval[float]: ...
     def overlaps(self: Interval[_OrderableT], other: Interval[_OrderableT]) -> bool: ...
 
 def intervals_to_interval_bounds(