Merge branch 'main' into 50040-add-math-mode-formatter-escape=latex-part2

Author: natmokval

.github/actions/build_pandas/action.yml

@@ -16,7 +16,5 @@ runs:
         python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
       shell: bash -el {0}
       env:
-        # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873
-        # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct
-        N_JOBS: 1
-        #N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }}
+        # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
+        N_JOBS: ${{ runner.os == 'macOS' && 3 || 2 }}

.github/actions/setup-conda/action.yml

@@ -30,7 +30,7 @@ runs:
         environment-name: ${{ inputs.environment-name }}
         extra-specs: ${{ inputs.extra-specs }}
         channels: conda-forge
-        channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
+        channel-priority: 'strict'
         condarc-file: ci/condarc.yml
         cache-env: true
         cache-downloads: true

.github/workflows/32-bit-linux.yml

@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
   pull_request:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
     paths-ignore:
       - "doc/**"
 

.github/workflows/code-checks.yml

@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
   pull_request:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
 
 env:
   ENV_FILE: environment.yml

.github/workflows/docbuild-and-upload.yml

@@ -5,14 +5,12 @@ on:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
 
 env:
   ENV_FILE: environment.yml

.github/workflows/macos-windows.yml

@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
   pull_request:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
     paths-ignore:
       - "doc/**"
       - "web/**"

.github/workflows/package-checks.yml

@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
   pull_request:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
     types: [ labeled, opened, synchronize, reopened ]
 
 permissions:

.github/workflows/python-dev.yml

@@ -23,13 +23,13 @@ name: Python Dev
 on:
   push:
     branches:
-#      - main
-#      - 1.5.x
+      - main
+      - 2.0.x
       - None
   pull_request:
     branches:
-#      - main
-#      - 1.5.x
+      - main
+      - 2.0.x
       - None
     paths-ignore:
       - "doc/**"
@@ -47,7 +47,7 @@ permissions:
 
 jobs:
   build:
-    # if: false # Uncomment this to freeze the workflow, comment it to unfreeze
+    if: false # Uncomment this to freeze the workflow, comment it to unfreeze
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.github/workflows/sdist.yml

@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
   pull_request:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
     types: [labeled, opened, synchronize, reopened]
     paths-ignore:
       - "doc/**"

.github/workflows/ubuntu.yml

@@ -5,12 +5,10 @@ on:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
   pull_request:
     branches:
       - main
       - 2.0.x
-      - 1.5.x
     paths-ignore:
       - "doc/**"
       - "web/**"

.pre-commit-config.yaml

@@ -31,6 +31,7 @@ repos:
     rev: v0.0.253
     hooks:
     -   id: ruff
+        args: [--exit-non-zero-on-fix]
 -   repo: https://github.com/jendrikseipp/vulture
     rev: 'v2.7'
     hooks:

MANIFEST.in

@@ -58,5 +58,3 @@ prune pandas/tests/io/parser/data
 # Selectively re-add *.cxx files that were excluded above
 graft pandas/_libs/src
 graft pandas/_libs/tslibs/src
-include pandas/_libs/pd_parser.h
-include pandas/_libs/pd_parser.c

asv_bench/benchmarks/algorithms.py

@@ -23,6 +23,7 @@ class Factorize:
             "uint",
             "float",
             "object",
+            "object_str",
             "datetime64[ns]",
             "datetime64[ns, tz]",
             "Int64",
@@ -46,7 +47,8 @@ def setup(self, unique, sort, dtype):
             "int": pd.Index(np.arange(N), dtype="int64"),
             "uint": pd.Index(np.arange(N), dtype="uint64"),
             "float": pd.Index(np.random.randn(N), dtype="float64"),
-            "object": string_index,
+            "object_str": string_index,
+            "object": pd.Index(np.arange(N), dtype="object"),
             "datetime64[ns]": pd.date_range("2011-01-01", freq="H", periods=N),
             "datetime64[ns, tz]": pd.date_range(
                 "2011-01-01", freq="H", periods=N, tz="Asia/Tokyo"
@@ -62,6 +64,9 @@ def setup(self, unique, sort, dtype):
     def time_factorize(self, unique, sort, dtype):
         pd.factorize(self.data, sort=sort)
 
+    def peakmem_factorize(self, unique, sort, dtype):
+        pd.factorize(self.data, sort=sort)
+
 
 class Duplicated:
     params = [

doc/source/development/contributing_environment.rst

@@ -21,7 +21,7 @@ locally before pushing your changes. It's recommended to also install the :ref:`
 Step 1: install a C compiler
 ----------------------------
 
-How to do this will depend on your platform. If you choose to user ``Docker``
+How to do this will depend on your platform. If you choose to use ``Docker``
 in the next step, then you can skip this step.
 
 **Windows**

doc/source/development/extending.rst

@@ -488,3 +488,49 @@ registers the default "matplotlib" backend as follows.
 
 More information on how to implement a third-party plotting backend can be found at
 https://github.com/pandas-dev/pandas/blob/main/pandas/plotting/__init__.py#L1.
+
+.. _extending.pandas_priority:
+
+Arithmetic with 3rd party types
+-------------------------------
+
+In order to control how arithmetic works between a custom type and a pandas type,
+implement ``__pandas_priority__``.  Similar to numpy's ``__array_priority__``
+semantics, arithmetic methods on :class:`DataFrame`, :class:`Series`, and :class:`Index`
+objects will delegate to ``other``, if it has an attribute ``__pandas_priority__`` with a higher value.
+
+By default, pandas objects try to operate with other objects, even if they are not types known to pandas:
+
+.. code-block:: python
+
+    >>> pd.Series([1, 2]) + [10, 20]
+    0    11
+    1    22
+    dtype: int64
+
+In the example above, if ``[10, 20]`` was a custom type that can be understood as a list, pandas objects will still operate with it in the same way.
+
+In some cases, it is useful to delegate to the other type the operation. For example, consider I implement a
+custom list object, and I want the result of adding my custom list with a pandas :class:`Series` to be an instance of my list
+and not a :class:`Series` as seen in the previous example. This is now possible by defining the ``__pandas_priority__`` attribute
+of my custom list, and setting it to a higher value, than the priority of the pandas objects I want to operate with.
+
+The ``__pandas_priority__`` of :class:`DataFrame`, :class:`Series`, and :class:`Index` are ``4000``, ``3000``, and ``2000`` respectively.  The base ``ExtensionArray.__pandas_priority__`` is ``1000``.
+
+.. code-block:: python
+
+    class CustomList(list):
+        __pandas_priority__ = 5000
+
+        def __radd__(self, other):
+            # return `self` and not the addition for simplicity
+            return self
+
+    custom = CustomList()
+    series = pd.Series([1, 2, 3])
+
+    # Series refuses to add custom, since it's an unknown type with higher priority
+    assert series.__add__(custom) is NotImplemented
+
+    # This will cause the custom class `__radd__` being used instead
+    assert series + custom is custom

doc/source/user_guide/integer_na.rst

@@ -32,7 +32,7 @@ implemented within pandas.
    arr = pd.array([1, 2, None], dtype=pd.Int64Dtype())
    arr
 
-Or the string alias ``"Int64"`` (note the capital ``"I"``, to differentiate from
+Or the string alias ``"Int64"`` (note the capital ``"I"``) to differentiate from
 NumPy's ``'int64'`` dtype:
 
 .. ipython:: python
@@ -67,7 +67,7 @@ with the dtype.
       pd.array([1, 2])
 
    For backwards-compatibility, :class:`Series` infers these as either
-   integer or float dtype
+   integer or float dtype.
 
    .. ipython:: python
 
@@ -101,7 +101,7 @@ dtype if needed.
    # comparison
    s == 1
 
-   # indexing
+   # slicing operation
    s.iloc[1:3]
 
    # operate with other dtypes
@@ -110,7 +110,7 @@ dtype if needed.
    # coerce when needed
    s + 0.01
 
-These dtypes can operate as part of ``DataFrame``.
+These dtypes can operate as part of a ``DataFrame``.
 
 .. ipython:: python
 
@@ -119,7 +119,7 @@ These dtypes can operate as part of ``DataFrame``.
    df.dtypes
 
 
-These dtypes can be merged & reshaped & casted.
+These dtypes can be merged, reshaped & casted.
 
 .. ipython:: python
 

doc/source/user_guide/io.rst

@@ -170,12 +170,15 @@ dtype : Type name or dict of column -> type, default ``None``
      the default determines the dtype of the columns which are not explicitly
      listed.
 
-use_nullable_dtypes : bool = False
-    Whether or not to use nullable dtypes as default when reading data. If
-    set to True, nullable dtypes are used for all dtypes that have a nullable
-    implementation, even if no nulls are present.
+dtype_backend : {"numpy_nullable", "pyarrow"}, defaults to NumPy backed DataFrames
+  Which dtype_backend to use, e.g. whether a DataFrame should have NumPy
+  arrays, nullable dtypes are used for all dtypes that have a nullable
+  implementation when "numpy_nullable" is set, pyarrow is used for all
+  dtypes if "pyarrow" is set.
 
-    .. versionadded:: 2.0
+  The dtype_backends are still experimential.
+
+  .. versionadded:: 2.0
 
 engine : {``'c'``, ``'python'``, ``'pyarrow'``}
   Parser engine to use. The C and pyarrow engines are faster, while the python engine
@@ -475,7 +478,7 @@ worth trying.
 
    os.remove("foo.csv")
 
-Setting ``use_nullable_dtypes=True`` will result in nullable dtypes for every column.
+Setting ``dtype_backend="numpy_nullable"`` will result in nullable dtypes for every column.
 
 .. ipython:: python
 
@@ -484,7 +487,7 @@ Setting ``use_nullable_dtypes=True`` will result in nullable dtypes for every co
    3,4.5,False,b,6,7.5,True,a,12-31-2019,
    """
 
-   df = pd.read_csv(StringIO(data), use_nullable_dtypes=True, parse_dates=["i"])
+   df = pd.read_csv(StringIO(data), dtype_backend="numpy_nullable", parse_dates=["i"])
    df
    df.dtypes
 

doc/source/user_guide/pyarrow.rst

@@ -145,8 +145,8 @@ functions provide an ``engine`` keyword that can dispatch to PyArrow to accelera
    df
 
 By default, these functions and all other IO reader functions return NumPy-backed data. These readers can return
-PyArrow-backed data by specifying the parameter ``use_nullable_dtypes=True`` **and** the global configuration option ``"mode.dtype_backend"``
-set to ``"pyarrow"``. A reader does not need to set ``engine="pyarrow"`` to necessarily return PyArrow-backed data.
+PyArrow-backed data by specifying the parameter ``dtype_backend="pyarrow"``. A reader does not need to set
+``engine="pyarrow"`` to necessarily return PyArrow-backed data.
 
 .. ipython:: python
 
@@ -155,20 +155,10 @@ set to ``"pyarrow"``. A reader does not need to set ``engine="pyarrow"`` to nece
         1,2.5,True,a,,,,,
         3,4.5,False,b,6,7.5,True,a,
     """)
-    with pd.option_context("mode.dtype_backend", "pyarrow"):
-        df_pyarrow = pd.read_csv(data, use_nullable_dtypes=True)
+    df_pyarrow = pd.read_csv(data, dtype_backend="pyarrow")
     df_pyarrow.dtypes
 
-To simplify specifying ``use_nullable_dtypes=True`` in several functions, you can set a global option ``nullable_dtypes``
-to ``True``. You will still need to set the global configuration option ``"mode.dtype_backend"`` to ``pyarrow``.
-
-.. code-block:: ipython
-
-    In [1]: pd.set_option("mode.dtype_backend", "pyarrow")
-
-    In [2]: pd.options.mode.nullable_dtypes = True
-
-Several non-IO reader functions can also use the ``"mode.dtype_backend"`` option to return PyArrow-backed data including:
+Several non-IO reader functions can also use the ``dtype_backend`` argument to return PyArrow-backed data including:
 
 * :func:`to_numeric`
 * :meth:`DataFrame.convert_dtypes`