Fix merge conflict in whatsnew

Author: MarcoGorelli

.gitignore

@@ -57,6 +57,7 @@ dist
 # wheel files
 *.whl
 **/wheelhouse/*
+pip-wheel-metadata
 # coverage
 .coverage
 coverage.xml

.pre-commit-config.yaml

@@ -15,7 +15,3 @@ repos:
     hooks:
     -   id: isort
         language: python_venv
--   repo: https://github.com/asottile/seed-isort-config
-    rev: v1.9.2
-    hooks:
-    -   id: seed-isort-config

MANIFEST.in

@@ -3,6 +3,7 @@ include LICENSE
 include RELEASE.md
 include README.md
 include setup.py
+include pyproject.toml
 
 graft doc
 prune doc/build

Makefile

@@ -18,7 +18,7 @@ black:
 	black . --exclude '(asv_bench/env|\.egg|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist|setup.py)'
 
 develop: build
-	python setup.py develop
+	python -m pip install --no-build-isolation -e .
 
 doc:
 	-rm -rf doc/build doc/source/generated

README.md

@@ -188,16 +188,17 @@ python setup.py install
 
 or for installing in [development mode](https://pip.pypa.io/en/latest/reference/pip_install.html#editable-installs):
 
+
 ```sh
-python setup.py develop
+python -m pip install --no-build-isolation -e .
 ```
 
-Alternatively, you can use `pip` if you want all the dependencies pulled
-in automatically (the `-e` option is for installing it in [development
-mode](https://pip.pypa.io/en/latest/reference/pip_install.html#editable-installs)):
+If you have `make`, you can also use `make develop` to run the same command.
+
+or alternatively
 
 ```sh
-pip install -e .
+python setup.py develop
 ```
 
 See the full instructions for [installing from source](https://pandas.pydata.org/pandas-docs/stable/install.html#installing-from-source).

asv_bench/asv.conf.json

@@ -50,12 +50,13 @@
         "xlsxwriter": [],
         "xlrd": [],
         "xlwt": [],
+        "odfpy": [],
         "pytest": [],
         // If using Windows with python 2.7 and want to build using the
         // mingw toolchain (rather than MSVC), uncomment the following line.
         // "libpython": [],
     },
-
+    "conda_channels": ["defaults", "conda-forge"],
     // Combinations of libraries/python versions can be excluded/included
     // from the set to test. Each entry is a dictionary containing additional
     // key-value pairs to include/exclude.

asv_bench/benchmarks/io/excel.py

@@ -1,40 +1,72 @@
 from io import BytesIO
 
 import numpy as np
+from odf.opendocument import OpenDocumentSpreadsheet
+from odf.table import Table, TableCell, TableRow
+from odf.text import P
 
 from pandas import DataFrame, ExcelWriter, date_range, read_excel
 import pandas.util.testing as tm
 
 
-class Excel:
+def _generate_dataframe():
+    N = 2000
+    C = 5
+    df = DataFrame(
+        np.random.randn(N, C),
+        columns=["float{}".format(i) for i in range(C)],
+        index=date_range("20000101", periods=N, freq="H"),
+    )
+    df["object"] = tm.makeStringIndex(N)
+    return df
+
+
+class WriteExcel:
 
     params = ["openpyxl", "xlsxwriter", "xlwt"]
     param_names = ["engine"]
 
     def setup(self, engine):
-        N = 2000
-        C = 5
-        self.df = DataFrame(
-            np.random.randn(N, C),
-            columns=["float{}".format(i) for i in range(C)],
-            index=date_range("20000101", periods=N, freq="H"),
-        )
-        self.df["object"] = tm.makeStringIndex(N)
-        self.bio_read = BytesIO()
-        self.writer_read = ExcelWriter(self.bio_read, engine=engine)
-        self.df.to_excel(self.writer_read, sheet_name="Sheet1")
-        self.writer_read.save()
-        self.bio_read.seek(0)
-
-    def time_read_excel(self, engine):
-        read_excel(self.bio_read)
+        self.df = _generate_dataframe()
 
     def time_write_excel(self, engine):
-        bio_write = BytesIO()
-        bio_write.seek(0)
-        writer_write = ExcelWriter(bio_write, engine=engine)
-        self.df.to_excel(writer_write, sheet_name="Sheet1")
-        writer_write.save()
+        bio = BytesIO()
+        bio.seek(0)
+        writer = ExcelWriter(bio, engine=engine)
+        self.df.to_excel(writer, sheet_name="Sheet1")
+        writer.save()
+
+
+class ReadExcel:
+
+    params = ["xlrd", "openpyxl", "odf"]
+    param_names = ["engine"]
+    fname_excel = "spreadsheet.xlsx"
+    fname_odf = "spreadsheet.ods"
+
+    def _create_odf(self):
+        doc = OpenDocumentSpreadsheet()
+        table = Table(name="Table1")
+        for row in self.df.values:
+            tr = TableRow()
+            for val in row:
+                tc = TableCell(valuetype="string")
+                tc.addElement(P(text=val))
+                tr.addElement(tc)
+            table.addElement(tr)
+
+        doc.spreadsheet.addElement(table)
+        doc.save(self.fname_odf)
+
+    def setup_cache(self):
+        self.df = _generate_dataframe()
+
+        self.df.to_excel(self.fname_excel, sheet_name="Sheet1")
+        self._create_odf()
+
+    def time_read_excel(self, engine):
+        fname = self.fname_odf if engine == "odf" else self.fname_excel
+        read_excel(fname, engine=engine)
 
 
 from ..pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/io/json.py

@@ -118,15 +118,15 @@ def setup(self, orient, frame):
     def time_to_json(self, orient, frame):
         getattr(self, frame).to_json(self.fname, orient=orient)
 
-    def mem_to_json(self, orient, frame):
+    def peakmem_to_json(self, orient, frame):
         getattr(self, frame).to_json(self.fname, orient=orient)
 
     def time_to_json_wide(self, orient, frame):
         base_df = getattr(self, frame).copy()
         df = concat([base_df.iloc[:100]] * 1000, ignore_index=True, axis=1)
         df.to_json(self.fname, orient=orient)
 
-    def mem_to_json_wide(self, orient, frame):
+    def peakmem_to_json_wide(self, orient, frame):
         base_df = getattr(self, frame).copy()
         df = concat([base_df.iloc[:100]] * 1000, ignore_index=True, axis=1)
         df.to_json(self.fname, orient=orient)

asv_bench/benchmarks/package.py

@@ -0,0 +1,25 @@
+"""
+Benchmarks for pandas at the package-level.
+"""
+import subprocess
+import sys
+
+from pandas.compat import PY37
+
+
+class TimeImport:
+    def time_import(self):
+        if PY37:
+            # on py37+ we the "-X importtime" usage gives us a more precise
+            #  measurement of the import time we actually care about,
+            #  without the subprocess or interpreter overhead
+            cmd = [sys.executable, "-X", "importtime", "-c", "import pandas as pd"]
+            p = subprocess.run(cmd, stderr=subprocess.PIPE)
+
+            line = p.stderr.splitlines()[-1]
+            field = line.split(b"|")[-2].strip()
+            total = int(field)  # microseconds
+            return total
+
+        cmd = [sys.executable, "-c", "import pandas as pd"]
+        subprocess.run(cmd, stderr=subprocess.PIPE)

asv_bench/benchmarks/rolling.py

@@ -21,6 +21,9 @@ def setup(self, constructor, window, dtype, method):
     def time_rolling(self, constructor, window, dtype, method):
         getattr(self.roll, method)()
 
+    def peakmem_rolling(self, constructor, window, dtype, method):
+        getattr(self.roll, method)()
+
 
 class ExpandingMethods:
 

asv_bench/benchmarks/stat_ops.py

@@ -113,12 +113,23 @@ def setup(self, method, use_bottleneck):
             nanops._USE_BOTTLENECK = use_bottleneck
         self.df = pd.DataFrame(np.random.randn(1000, 30))
         self.df2 = pd.DataFrame(np.random.randn(1000, 30))
+        self.df_wide = pd.DataFrame(np.random.randn(1000, 200))
+        self.df_wide_nans = self.df_wide.where(np.random.random((1000, 200)) < 0.9)
         self.s = pd.Series(np.random.randn(1000))
         self.s2 = pd.Series(np.random.randn(1000))
 
     def time_corr(self, method, use_bottleneck):
         self.df.corr(method=method)
 
+    def time_corr_wide(self, method, use_bottleneck):
+        self.df_wide.corr(method=method)
+
+    def time_corr_wide_nans(self, method, use_bottleneck):
+        self.df_wide_nans.corr(method=method)
+
+    def peakmem_corr_wide(self, method, use_bottleneck):
+        self.df_wide.corr(method=method)
+
     def time_corr_series(self, method, use_bottleneck):
         self.s.corr(self.s2, method=method)
 

ci/code_checks.sh

@@ -203,10 +203,14 @@ if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then
 import sys
 import pandas
 
-blacklist = {'bs4', 'gcsfs', 'html5lib', 'ipython', 'jinja2' 'hypothesis',
+blacklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
              'lxml', 'numexpr', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
-             'tables', 'xlrd', 'xlsxwriter', 'xlwt'}
-mods = blacklist & set(m.split('.')[0] for m in sys.modules)
+             'tables', 'urllib.request', 'xlrd', 'xlsxwriter', 'xlwt'}
+
+# GH#28227 for some of these check for top-level modules, while others are
+#  more specific (e.g. urllib.request)
+import_mods = set(m.split('.')[0] for m in sys.modules) | set(sys.modules)
+mods = blacklist & import_mods
 if mods:
     sys.stderr.write('err: pandas should not import: {}\n'.format(', '.join(mods)))
     sys.exit(len(mods))

ci/deps/azure-35-compat.yaml

@@ -18,13 +18,13 @@ dependencies:
   - xlsxwriter=0.9.8
   - xlwt=1.2.0
   # universal
-  - cython=0.28.2
   - hypothesis>=3.58.0
   - pytest-xdist
   - pytest-mock
   - pytest-azurepipelines
   - pip
   - pip:
-    # for python 3.5, pytest>=4.0.2 is not available in conda
+    # for python 3.5, pytest>=4.0.2, cython>=0.29.13 is not available in conda
+    - cython>=0.29.13
     - pytest==4.5.0
     - html5lib==1.0b2

ci/deps/azure-36-32bit.yaml

@@ -6,7 +6,6 @@ dependencies:
   - gcc_linux-32
   - gcc_linux-32
   - gxx_linux-32
-  - cython=0.28.2
   - numpy=1.14.*
   - python-dateutil
   - python=3.6.*
@@ -18,3 +17,6 @@ dependencies:
   - pytest-azurepipelines
   - hypothesis>=3.58.0
   - pip
+  - pip:
+    # Anaconda doesn't build a new enough Cython
+    - cython>=0.29.13

ci/deps/azure-36-locale.yaml

@@ -5,7 +5,7 @@ channels:
 dependencies:
   - beautifulsoup4==4.6.0
   - bottleneck=1.2.*
-  - cython=0.28.2
+  - cython=0.29.13
   - lxml
   - matplotlib=2.2.2
   - numpy=1.14.*

ci/deps/azure-36-locale_slow.yaml

@@ -4,7 +4,7 @@ channels:
   - conda-forge
 dependencies:
   - beautifulsoup4
-  - cython>=0.28.2
+  - cython>=0.29.13
   - gcsfs
   - html5lib
   - ipython

ci/deps/azure-37-locale.yaml

@@ -4,7 +4,7 @@ channels:
   - conda-forge
 dependencies:
   - beautifulsoup4
-  - cython>=0.28.2
+  - cython>=0.29.13
   - html5lib
   - ipython
   - jinja2

ci/deps/azure-37-numpydev.yaml

@@ -4,7 +4,7 @@ channels:
 dependencies:
   - python=3.7.*
   - pytz
-  - Cython>=0.28.2
+  - Cython>=0.29.13
   # universal
   # pytest < 5 until defaults has pytest-xdist>=1.29.0
   - pytest>=4.0.2,<5.0

ci/deps/azure-macos-35.yaml

@@ -4,7 +4,6 @@ channels:
 dependencies:
   - beautifulsoup4
   - bottleneck
-  - cython>=0.28.2
   - html5lib
   - jinja2
   - lxml
@@ -24,6 +23,8 @@ dependencies:
   - xlwt
   - pip
   - pip:
+    # Anaconda / conda-forge don't build for 3.5
+    - cython>=0.29.13
     - pyreadstat
     # universal
     - pytest>=5.0.1
@@ -32,4 +33,3 @@ dependencies:
     - hypothesis>=3.58.0
     # https://github.com/pandas-dev/pandas/issues/27421
     - pytest-azurepipelines<1.0.0
-

ci/deps/azure-windows-36.yaml

@@ -1,17 +1,15 @@
 name: pandas-dev
 channels:
-  - defaults
   - conda-forge
+  - defaults
 dependencies:
   - blosc
   - bottleneck
-  - boost-cpp<1.67
   - fastparquet>=0.2.1
   - matplotlib=3.0.2
   - numexpr
   - numpy=1.15.*
   - openpyxl
-  - parquet-cpp
   - pyarrow
   - pytables
   - python-dateutil
@@ -22,7 +20,7 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - cython>=0.28.2
+  - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.29.0
   - pytest-mock

ci/deps/azure-windows-37.yaml

@@ -25,7 +25,7 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - cython>=0.28.2
+  - cython>=0.29.13
   - pytest>=5.0.0
   - pytest-xdist>=1.29.0
   - pytest-mock

ci/deps/travis-36-cov.yaml

@@ -5,7 +5,7 @@ channels:
 dependencies:
   - beautifulsoup4
   - botocore>=1.11
-  - cython>=0.28.2
+  - cython>=0.29.13
   - dask
   - fastparquet>=0.2.1
   - gcsfs