Merge remote-tracking branch 'upstream/master' into apply_output_cleanup

Author: rhshadrach

.travis.yml

@@ -45,7 +45,7 @@ matrix:
         - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
-        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
+        - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
       services:
         - mysql
         - postgresql
@@ -54,7 +54,7 @@ matrix:
         # Enabling Deprecations when running tests
         # PANDAS_TESTING_MODE="deprecate" causes DeprecationWarning messages to be displayed in the logs
         # See pandas/_testing.py for more details.
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
+        - JOB="3.7, coverage" ENV_FILE="ci/deps/travis-37-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
       services:
         - mysql
         - postgresql

asv_bench/benchmarks/index_object.py

@@ -57,8 +57,8 @@ def time_datetime_difference_disjoint(self):
 
 class Range:
     def setup(self):
-        self.idx_inc = RangeIndex(start=0, stop=10 ** 7, step=3)
-        self.idx_dec = RangeIndex(start=10 ** 7, stop=-1, step=-3)
+        self.idx_inc = RangeIndex(start=0, stop=10 ** 6, step=3)
+        self.idx_dec = RangeIndex(start=10 ** 6, stop=-1, step=-3)
 
     def time_max(self):
         self.idx_inc.max()
@@ -73,15 +73,23 @@ def time_min_trivial(self):
         self.idx_inc.min()
 
     def time_get_loc_inc(self):
-        self.idx_inc.get_loc(900000)
+        self.idx_inc.get_loc(900_000)
 
     def time_get_loc_dec(self):
-        self.idx_dec.get_loc(100000)
+        self.idx_dec.get_loc(100_000)
+
+    def time_iter_inc(self):
+        for _ in self.idx_inc:
+            pass
+
+    def time_iter_dec(self):
+        for _ in self.idx_dec:
+            pass
 
 
 class IndexEquals:
     def setup(self):
-        idx_large_fast = RangeIndex(100000)
+        idx_large_fast = RangeIndex(100_000)
         idx_small_slow = date_range(start="1/1/2012", periods=1)
         self.mi_large_slow = MultiIndex.from_product([idx_large_fast, idx_small_slow])
 
@@ -94,7 +102,7 @@ def time_non_object_equals_multiindex(self):
 class IndexAppend:
     def setup(self):
 
-        N = 10000
+        N = 10_000
         self.range_idx = RangeIndex(0, 100)
         self.int_idx = self.range_idx.astype(int)
         self.obj_idx = self.int_idx.astype(str)
@@ -168,7 +176,7 @@ def time_get_loc_non_unique_sorted(self, dtype):
 class Float64IndexMethod:
     # GH 13166
     def setup(self):
-        N = 100000
+        N = 100_000
         a = np.arange(N)
         self.ind = Float64Index(a * 4.8000000418824129e-08)
 
@@ -212,7 +220,7 @@ class GC:
     params = [1, 2, 5]
 
     def create_use_drop(self):
-        idx = Index(list(range(1000 * 1000)))
+        idx = Index(list(range(1_000_000)))
         idx._engine
 
     def peakmem_gc_instances(self, N):

asv_bench/benchmarks/package.py

@@ -4,22 +4,16 @@
 import subprocess
 import sys
 
-from pandas.compat import PY37
-
 
 class TimeImport:
     def time_import(self):
-        if PY37:
-            # on py37+ we the "-X importtime" usage gives us a more precise
-            #  measurement of the import time we actually care about,
-            #  without the subprocess or interpreter overhead
-            cmd = [sys.executable, "-X", "importtime", "-c", "import pandas as pd"]
-            p = subprocess.run(cmd, stderr=subprocess.PIPE)
-
-            line = p.stderr.splitlines()[-1]
-            field = line.split(b"|")[-2].strip()
-            total = int(field)  # microseconds
-            return total
+        # on py37+ we the "-X importtime" usage gives us a more precise
+        #  measurement of the import time we actually care about,
+        #  without the subprocess or interpreter overhead
+        cmd = [sys.executable, "-X", "importtime", "-c", "import pandas as pd"]
+        p = subprocess.run(cmd, stderr=subprocess.PIPE)
 
-        cmd = [sys.executable, "-c", "import pandas as pd"]
-        subprocess.run(cmd, stderr=subprocess.PIPE)
+        line = p.stderr.splitlines()[-1]
+        field = line.split(b"|")[-2].strip()
+        total = int(field)  # microseconds
+        return total

ci/azure/posix.yml

@@ -9,57 +9,57 @@ jobs:
   strategy:
     matrix:
       ${{ if eq(parameters.name, 'macOS') }}:
-        py36_macos:
-          ENV_FILE: ci/deps/azure-macos-36.yaml
-          CONDA_PY: "36"
+        py37_macos:
+          ENV_FILE: ci/deps/azure-macos-37.yaml
+          CONDA_PY: "37"
           PATTERN: "not slow and not network"
 
       ${{ if eq(parameters.name, 'Linux') }}:
-        py36_minimum_versions:
-          ENV_FILE: ci/deps/azure-36-minimum_versions.yaml
-          CONDA_PY: "36"
+        py37_minimum_versions:
+          ENV_FILE: ci/deps/azure-37-minimum_versions.yaml
+          CONDA_PY: "37"
           PATTERN: "not slow and not network and not clipboard"
 
-        py36_locale_slow_old_np:
-          ENV_FILE: ci/deps/azure-36-locale_slow.yaml
-          CONDA_PY: "36"
+        py37_locale_slow:
+          ENV_FILE: ci/deps/azure-37-locale_slow.yaml
+          CONDA_PY: "37"
           PATTERN: "slow"
           # pandas does not use the language (zh_CN), but should support different encodings (utf8)
           # we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any
           LANG: "zh_CN.utf8"
           LC_ALL: "zh_CN.utf8"
           EXTRA_APT: "language-pack-zh-hans"
 
-        py36_slow:
-          ENV_FILE: ci/deps/azure-36-slow.yaml
-          CONDA_PY: "36"
+        py37_slow:
+          ENV_FILE: ci/deps/azure-37-slow.yaml
+          CONDA_PY: "37"
           PATTERN: "slow"
 
-        py36_locale:
-          ENV_FILE: ci/deps/azure-36-locale.yaml
-          CONDA_PY: "36"
+        py37_locale:
+          ENV_FILE: ci/deps/azure-37-locale.yaml
+          CONDA_PY: "37"
           PATTERN: "not slow and not network"
           LANG: "it_IT.utf8"
           LC_ALL: "it_IT.utf8"
           EXTRA_APT: "language-pack-it xsel"
 
-        #py36_32bit:
-        #  ENV_FILE: ci/deps/azure-36-32bit.yaml
-        #  CONDA_PY: "36"
-        #  PATTERN: "not slow and not network and not clipboard"
-        #  BITS32: "yes"
+#        py37_32bit:
+#          ENV_FILE: ci/deps/azure-37-32bit.yaml
+#          CONDA_PY: "37"
+#          PATTERN: "not slow and not network and not clipboard"
+#          BITS32: "yes"
 
-        py37_locale:
-          ENV_FILE: ci/deps/azure-37-locale.yaml
-          CONDA_PY: "37"
+        py38_locale:
+          ENV_FILE: ci/deps/azure-38-locale.yaml
+          CONDA_PY: "38"
           PATTERN: "not slow and not network"
           LANG: "zh_CN.utf8"
           LC_ALL: "zh_CN.utf8"
           EXTRA_APT: "language-pack-zh-hans xsel"
 
-        py37_np_dev:
-          ENV_FILE: ci/deps/azure-37-numpydev.yaml
-          CONDA_PY: "37"
+        py38_np_dev:
+          ENV_FILE: ci/deps/azure-38-numpydev.yaml
+          CONDA_PY: "38"
           PATTERN: "not slow and not network"
           TEST_ARGS: "-W error"
           PANDAS_TESTING_MODE: "deprecate"

ci/azure/windows.yml

@@ -8,16 +8,16 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py36_np15:
-        ENV_FILE: ci/deps/azure-windows-36.yaml
-        CONDA_PY: "36"
-        PATTERN: "not slow and not network"
-
-      py37_np18:
+      py37_np16:
         ENV_FILE: ci/deps/azure-windows-37.yaml
         CONDA_PY: "37"
         PATTERN: "not slow and not network"
 
+      py38_np18:
+        ENV_FILE: ci/deps/azure-windows-38.yaml
+        CONDA_PY: "38"
+        PATTERN: "not slow and not network"
+
   steps:
     - powershell: |
         Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"

ci/code_checks.sh

@@ -121,7 +121,7 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
 
     # Imports - Check formatting using isort see setup.cfg for settings
     MSG='Check import format using isort' ; echo $MSG
-    ISORT_CMD="isort --quiet --check-only pandas asv_bench scripts"
+    ISORT_CMD="isort --quiet --check-only pandas asv_bench scripts web"
     if [[ "$GITHUB_ACTIONS" == "true" ]]; then
         eval $ISORT_CMD | awk '{print "##[error]" $0}'; RET=$(($RET + ${PIPESTATUS[0]}))
     else
@@ -230,6 +230,11 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include="*.py" -P '# type: (?!ignore)' pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    # https://github.com/python/mypy/issues/7384
+    # MSG='Check for missing error codes with # type: ignore' ; echo $MSG
+    # invgrep -R --include="*.py" -P '# type: ignore(?!\[)' pandas
+    # RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Check for use of foo.__class__ instead of type(foo)' ; echo $MSG
     invgrep -R --include=*.{py,pyx} '\.__class__' pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"

ci/deps/azure-36-32bit.yaml renamed to ci/deps/azure-37-32bit.yaml

@@ -3,10 +3,10 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - python=3.6.*
+  - python=3.7.*
 
   # tools
-  ### Cython 0.29.13 and pytest 5.0.1 for 32 bits are not available with conda, installing below with pip instead
+  ### Cython 0.29.16 and pytest 5.0.1 for 32 bits are not available with conda, installing below with pip instead
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
   - pytest-azurepipelines
@@ -15,12 +15,12 @@ dependencies:
   - attrs=19.1.0
   - gcc_linux-32
   - gxx_linux-32
-  - numpy=1.14.*
   - python-dateutil
-  - pytz=2017.2
+  - pytz=2017.3
 
   # see comment above
   - pip
   - pip:
     - cython>=0.29.16
+    - numpy>=1.16.5
     - pytest>=5.0.1

ci/deps/azure-37-locale.yaml

@@ -1,5 +1,6 @@
 name: pandas-dev
 channels:
+  - defaults
   - conda-forge
 dependencies:
   - python=3.7.*
@@ -22,7 +23,7 @@ dependencies:
   - moto
   - nomkl
   - numexpr
-  - numpy
+  - numpy=1.16.*
   - openpyxl
   - pytables
   - python-dateutil
@@ -32,7 +33,4 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
-  - pyarrow>=0.15
-  - pip
-  - pip:
-    - pyxlsb
+  - moto

ci/deps/azure-36-locale_slow.yaml renamed to ci/deps/azure-37-locale_slow.yaml

@@ -3,7 +3,7 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - python=3.6.*
+  - python=3.7.*
 
   # tools
   - cython>=0.29.16
@@ -16,17 +16,15 @@ dependencies:
   - beautifulsoup4=4.6.0
   - bottleneck=1.2.*
   - lxml
-  - matplotlib=2.2.2
-  - numpy=1.14.*
+  - matplotlib=3.0.0
+  - numpy=1.16.*
   - openpyxl=2.5.7
   - python-dateutil
   - python-blosc
-  - pytz=2017.2
+  - pytz=2017.3
   - scipy
-  - sqlalchemy=1.1.4
+  - sqlalchemy=1.2.8
   - xlrd=1.1.0
-  - xlsxwriter=0.9.8
-  - xlwt=1.2.0
-  - pip
-  - pip:
-    - html5lib==1.0b2
+  - xlsxwriter=1.0.2
+  - xlwt=1.3.0
+  - html5lib=1.0.1

ci/deps/azure-36-minimum_versions.yaml renamed to ci/deps/azure-37-minimum_versions.yaml

@@ -2,7 +2,7 @@ name: pandas-dev
 channels:
   - conda-forge
 dependencies:
-  - python=3.6.1
+  - python=3.7.1
 
   # tools
   - cython=0.29.16
@@ -15,16 +15,17 @@ dependencies:
   # pandas dependencies
   - beautifulsoup4=4.6.0
   - bottleneck=1.2.1
-  - jinja2=2.8
+  - jinja2=2.10
   - numba=0.46.0
-  - numexpr=2.6.2
-  - numpy=1.15.4
+  - numexpr=2.6.8
+  - numpy=1.16.5
   - openpyxl=2.5.7
-  - pytables=3.4.3
+  - pytables=3.4.4
   - python-dateutil=2.7.3
-  - pytz=2017.2
+  - pytz=2017.3
+  - pyarrow=0.15
   - scipy=1.2
   - xlrd=1.1.0
-  - xlsxwriter=0.9.8
-  - xlwt=1.2.0
+  - xlsxwriter=1.0.2
+  - xlwt=1.3.0
   - html5lib=1.0.1

ci/deps/azure-36-slow.yaml renamed to ci/deps/azure-37-slow.yaml

@@ -3,7 +3,7 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - python=3.6.*
+  - python=3.7.*
 
   # tools
   - cython>=0.29.16