Merge remote-tracking branch 'upstream/master' into test_astype_string

Author: simonjayhawkins

.pre-commit-config.yaml

@@ -1,5 +1,7 @@
 minimum_pre_commit_version: 2.9.2
 exclude: ^LICENSES/|\.(html|csv|svg)$
+ci:
+    autofix_prs: false
 repos:
 -   repo: https://github.com/MarcoGorelli/absolufy-imports
     rev: v0.3.0
@@ -36,7 +38,10 @@ repos:
     rev: 3.9.0
     hooks:
     -   id: flake8
-        additional_dependencies: [flake8-comprehensions>=3.1.0]
+        additional_dependencies:
+            - flake8-comprehensions==3.1.0
+            - flake8-bugbear==21.3.2
+            - pandas-dev-flaker==0.2.0
     -   id: flake8
         name: flake8 (cython)
         types: [cython]
@@ -51,10 +56,10 @@ repos:
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.11.0
+    rev: v2.12.0
     hooks:
     -   id: pyupgrade
-        args: [--py37-plus, --keep-runtime-typing]
+        args: [--py37-plus]
 -   repo: https://github.com/pre-commit/pygrep-hooks
     rev: v1.8.0
     hooks:
@@ -69,6 +74,11 @@ repos:
     rev: v1.2.2
     hooks:
     -   id: yesqa
+        additional_dependencies:
+            - flake8==3.9.0
+            - flake8-comprehensions==3.1.0
+            - flake8-bugbear==21.3.2
+            - pandas-dev-flaker==0.2.0
 -   repo: local
     hooks:
     -   id: flake8-rst
@@ -79,77 +89,18 @@ repos:
         types: [rst]
         args: [--filename=*.rst]
         additional_dependencies: [flake8-rst==0.7.0, flake8==3.7.9]
-    -   id: frame-or-series-union
-        name: Check for use of Union[Series, DataFrame] instead of FrameOrSeriesUnion alias
-        entry: Union\[.*(Series,.*DataFrame|DataFrame,.*Series).*\]
-        language: pygrep
-        types: [python]
-        exclude: ^pandas/_typing\.py$
-    -   id: inconsistent-namespace-usage
-        name: 'Check for inconsistent use of pandas namespace in tests'
-        entry: python scripts/check_for_inconsistent_pandas_namespace.py
-        language: python
-        types: [python]
-        files: ^pandas/tests/
-    -   id: incorrect-code-directives
-        name: Check for incorrect code block or IPython directives
-        language: pygrep
-        entry: (\.\. code-block ::|\.\. ipython ::)
-        types_or: [python, cython, rst]
-    -   id: no-os-remove
-        name: Check code for instances of os.remove
-        entry: os\.remove
-        language: pygrep
-        types: [python]
-        files: ^pandas/tests/
-        exclude: |
-            (?x)^
-            pandas/tests/io/excel/test_writers\.py
-            |pandas/tests/io/pytables/common\.py
-            |pandas/tests/io/pytables/test_store\.py$
-    -   id: non-standard-imports
-        name: Check for non-standard imports
-        language: pygrep
-        entry: |
-            (?x)
-            # Check for imports from pandas.core.common instead of `import pandas.core.common as com`
-            from\ pandas\.core\.common\ import
-            |from\ pandas\.core\ import\ common
-
-            # Check for imports from collections.abc instead of `from collections import abc`
-            |from\ collections\.abc\ import
-
-            # Numpy
-            |from\ numpy\ import\ random
-            |from\ numpy\.random\ import
-        types: [python]
-    -   id: non-standard-imports-in-tests
-        name: Check for non-standard imports in test suite
+    -   id: unwanted-patterns
+        name: Unwanted patterns
         language: pygrep
         entry: |
             (?x)
-            # Check for imports from pandas._testing instead of `import pandas._testing as tm`
-            from\ pandas\._testing\ import
-            |from\ pandas\ import\ _testing\ as\ tm
-
-            # No direct imports from conftest
-            |conftest\ import
-            |import\ conftest
-
-            # Check for use of pandas.testing instead of tm
-            |pd\.testing\.
+            # outdated annotation syntax, missing error codes
+            \#\ type:\ (?!ignore)
+            |\#\ type:\s?ignore(?!\[)
 
-            # Check for pd.api.types instead of from pandas.api.types import ...
-            |(pd|pandas)\.api\.types\.
-        types: [python]
-        files: ^pandas/tests/
-    -   id: np-bool-and-np-object
-        name: Check for use of np.bool/np.object instead of np.bool_/np.object_
-        entry: |
-            (?x)
-            np\.bool[^_8]
-            |np\.object[^_8]
-        language: pygrep
+            # Incorrect code-block / IPython directives
+            |\.\.\ code-block\ ::
+            |\.\.\ ipython\ ::
         types_or: [python, cython, rst]
     -   id: pip-to-conda
         name: Generate pip dependency from conda
@@ -159,57 +110,28 @@ repos:
         files: ^(environment.yml|requirements-dev.txt)$
         pass_filenames: false
         additional_dependencies: [pyyaml]
+    -   id: sync-flake8-versions
+        name: Check flake8 version is synced across flake8, yesqa, and environment.yml
+        language: python
+        entry: python scripts/sync_flake8_versions.py
+        files: ^(\.pre-commit-config\.yaml|environment\.yml)$
+        pass_filenames: false
+        additional_dependencies: [pyyaml]
     -   id: title-capitalization
         name: Validate correct capitalization among titles in documentation
         entry: python scripts/validate_rst_title_capitalization.py
         language: python
         types: [rst]
         files: ^doc/source/(development|reference)/
-    -   id: type-not-class
-        name: Check for use of foo.__class__ instead of type(foo)
-        entry: \.__class__
-        language: pygrep
-        types_or: [python, cython]
-    -   id: unwanted-patterns-bare-pytest-raises
-        name: Check for use of bare pytest raises
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="bare_pytest_raises"
-        types: [python]
-        files: ^pandas/tests/
-        exclude: ^pandas/tests/extension/
-    -   id: unwanted-patterns-private-function-across-module
-        name: Check for use of private functions across modules
+    -   id: use-pd_array-in-core
+        name: Import pandas.array as pd_array in core
         language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="private_function_across_module"
+        entry: python scripts/use_pd_array_in_core.py
+        files: ^pandas/core/
+        exclude: ^pandas/core/api\.py$
         types: [python]
-        exclude: ^(asv_bench|pandas/tests|doc)/
-    -   id: unwanted-patterns-private-import-across-module
-        name: Check for import of private attributes across modules
+    -   id: no-bool-in-core-generic
+        name: Use bool_t instead of bool in pandas/core/generic.py
+        entry: python scripts/no_bool_in_generic.py
         language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="private_import_across_module"
-        types: [python]
-        exclude: ^(asv_bench|pandas/tests|doc)/
-    -   id: unwanted-patterns-pytest-xfail
-        name: Check for use of pytest.xfail
-        entry: pytest\.xfail
-        language: pygrep
-        types: [python]
-        files: ^pandas/tests/
-    -   id: unwanted-patterns-strings-to-concatenate
-        name: Check for use of not concatenated strings
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_to_concatenate"
-        types_or: [python, cython]
-    -   id: unwanted-patterns-strings-with-wrong-placed-whitespace
-        name: Check for strings with wrong placed spaces
-        language: python
-        entry: python scripts/validate_unwanted_patterns.py --validation-type="strings_with_wrong_placed_whitespace"
-        types_or: [python, cython]
-    -   id: unwanted-typing
-        name: Check for outdated annotation syntax and missing error codes
-        entry: |
-            (?x)
-            \#\ type:\ (?!ignore)
-            |\#\ type:\s?ignore(?!\[)
-        language: pygrep
-        types: [python]
+        files: ^pandas/core/generic\.py$

.travis.yml

@@ -35,16 +35,11 @@ matrix:
   fast_finish: true
 
   include:
-    - arch: arm64
+    - arch: arm64-graviton2
+      virt: lxd
+      group: edge
       env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
-  allow_failures:
-    # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719
-    - arch: arm64
-      env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
-
+        - JOB="3.7, arm64" PYTEST_WORKERS="auto" ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
 before_install:
   - echo "before_install"

LICENSES/PYUPGRADE_LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2017 Anthony Sottile
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

MANIFEST.in

@@ -17,8 +17,10 @@ global-exclude *.h5
 global-exclude *.html
 global-exclude *.json
 global-exclude *.jsonl
+global-exclude *.pdf
 global-exclude *.pickle
 global-exclude *.png
+global-exclude *.pptx
 global-exclude *.pyc
 global-exclude *.pyd
 global-exclude *.ods

asv_bench/benchmarks/arithmetic.py

@@ -140,9 +140,7 @@ def setup(self, op, shape):
         # construct dataframe with 2 blocks
         arr1 = np.random.randn(n_rows, n_cols // 2).astype("f8")
         arr2 = np.random.randn(n_rows, n_cols // 2).astype("f4")
-        df = pd.concat(
-            [pd.DataFrame(arr1), pd.DataFrame(arr2)], axis=1, ignore_index=True
-        )
+        df = pd.concat([DataFrame(arr1), DataFrame(arr2)], axis=1, ignore_index=True)
         # should already be the case, but just to be sure
         df._consolidate_inplace()
 
@@ -151,7 +149,7 @@ def setup(self, op, shape):
         arr2 = np.random.randn(n_rows, n_cols // 2).astype("i8")
         arr3 = np.random.randn(n_rows, n_cols // 4).astype("f8")
         df2 = pd.concat(
-            [pd.DataFrame(arr1), pd.DataFrame(arr2), pd.DataFrame(arr3)],
+            [DataFrame(arr1), DataFrame(arr2), DataFrame(arr3)],
             axis=1,
             ignore_index=True,
         )
@@ -459,9 +457,9 @@ class OffsetArrayArithmetic:
 
     def setup(self, offset):
         N = 10000
-        rng = pd.date_range(start="1/1/2000", periods=N, freq="T")
+        rng = date_range(start="1/1/2000", periods=N, freq="T")
         self.rng = rng
-        self.ser = pd.Series(rng)
+        self.ser = Series(rng)
 
     def time_add_series_offset(self, offset):
         with warnings.catch_warnings(record=True):
@@ -478,7 +476,7 @@ class ApplyIndex:
 
     def setup(self, offset):
         N = 10000
-        rng = pd.date_range(start="1/1/2000", periods=N, freq="T")
+        rng = date_range(start="1/1/2000", periods=N, freq="T")
         self.rng = rng
 
     def time_apply_index(self, offset):
@@ -490,17 +488,17 @@ class BinaryOpsMultiIndex:
     param_names = ["func"]
 
     def setup(self, func):
-        date_range = pd.date_range("20200101 00:00", "20200102 0:00", freq="S")
+        array = date_range("20200101 00:00", "20200102 0:00", freq="S")
         level_0_names = [str(i) for i in range(30)]
 
-        index = pd.MultiIndex.from_product([level_0_names, date_range])
+        index = pd.MultiIndex.from_product([level_0_names, array])
         column_names = ["col_1", "col_2"]
 
-        self.df = pd.DataFrame(
+        self.df = DataFrame(
             np.random.rand(len(index), 2), index=index, columns=column_names
         )
 
-        self.arg_df = pd.DataFrame(
+        self.arg_df = DataFrame(
             np.random.randint(1, 10, (len(level_0_names), 2)),
             index=level_0_names,
             columns=column_names,

asv_bench/benchmarks/gil.py

@@ -31,7 +31,7 @@
 except ImportError:
     from pandas import algos
 try:
-    from pandas._testing import test_parallel
+    from pandas._testing import test_parallel  # noqa: PDF014
 
     have_real_test_parallel = True
 except ImportError:

asv_bench/benchmarks/groupby.py

@@ -480,7 +480,19 @@ class GroupByCythonAgg:
     param_names = ["dtype", "method"]
     params = [
         ["float64"],
-        ["sum", "prod", "min", "max", "mean", "median", "var", "first", "last"],
+        [
+            "sum",
+            "prod",
+            "min",
+            "max",
+            "mean",
+            "median",
+            "var",
+            "first",
+            "last",
+            "any",
+            "all",
+        ],
     ]
 
     def setup(self, dtype, method):

asv_bench/benchmarks/pandas_vb_common.py

@@ -70,7 +70,7 @@ class BaseIO:
     def remove(self, f):
         """Remove created files"""
         try:
-            os.remove(f)
+            os.remove(f)  # noqa: PDF008
         except OSError:
             # On Windows, attempting to remove a file that is in use
             # causes an exception to be raised

asv_bench/benchmarks/sparse.py

@@ -28,7 +28,7 @@ def setup(self):
             data = np.random.randn(N)[:-i]
             idx = rng[:-i]
             data[100:] = np.nan
-            self.series[i] = pd.Series(pd.SparseArray(data), index=idx)
+            self.series[i] = Series(SparseArray(data), index=idx)
 
     def time_series_to_frame(self):
         pd.DataFrame(self.series)
@@ -63,7 +63,7 @@ def setup(self):
         )
 
     def time_sparse_series_from_coo(self):
-        pd.Series.sparse.from_coo(self.matrix)
+        Series.sparse.from_coo(self.matrix)
 
 
 class ToCoo:

ci/azure/windows.yml

@@ -8,7 +8,7 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py37_np16:
+      py37_np17:
         ENV_FILE: ci/deps/azure-windows-37.yaml
         CONDA_PY: "37"
         PATTERN: "not slow and not network"