Merge remote-tracking branch 'upstream/master' into pivot

* upstream/master: (110 commits)
  DOC: hardcode contributors for 0.24.x releases (pandas-dev#25662)
  DOC: restore toctree maxdepth (pandas-dev#25134)
  BUG: Redefine IndexOpsMixin.size, fix pandas-dev#25580. (pandas-dev#25584)
  BUG: to_csv line endings with compression (pandas-dev#25625)
  DOC: file obj for to_csv must be newline='' (pandas-dev#25624)
  Suppress incorrect warning in nargsort for timezone-aware DatetimeIndex (pandas-dev#25629)
  TST: fix incorrect sparse test (now failing on scipy master) (pandas-dev#25653)
  CLN: Removed debugging code (pandas-dev#25647)
  DOC: require Return section only if return is not None nor commentary (pandas-dev#25008)
  DOC:Remove hard-coded examples from _flex_doc_SERIES (pandas-dev#24589) (pandas-dev#25524)
  TST: xref pandas-dev#25630 (pandas-dev#25643)
  BUG: Fix pandas-dev#25481 by fixing the error message in TypeError (pandas-dev#25540)
  Fixturize tests/frame/test_mutate_columns.py (pandas-dev#25642)
  Fixturize tests/frame/test_join.py (pandas-dev#25639)
  Fixturize tests/frame/test_combine_concat.py (pandas-dev#25634)
  Fixturize tests/frame/test_asof.py (pandas-dev#25628)
  BUG: Fix user-facing AssertionError with to_html (pandas-dev#25608) (pandas-dev#25620)
  DOC: resolve all GL03 docstring validation errors (pandas-dev#25525)
  TST: failing wheel building on PY2 and old numpy (pandas-dev#25631)
  DOC: Remove makePanel from docs (pandas-dev#25609) (pandas-dev#25612)
  ...

Author: thoo

README.md

@@ -26,8 +26,9 @@
   <td>Package Status</td>
   <td>
 		<a href="https://pypi.org/project/pandas/">
-		<img src="https://img.shields.io/pypi/status/pandas.svg" alt="status" /></td>
+		<img src="https://img.shields.io/pypi/status/pandas.svg" alt="status" />
 		</a>
+  </td>
 </tr>
 <tr>
   <td>License</td>
@@ -73,8 +74,8 @@
 	<td>Gitter</td>
 	<td>
 		<a href="https://gitter.im/pydata/pandas">
-		<img src="https://badges.gitter.im/Join%20Chat.svg"
-	</a>
+		<img src="https://badges.gitter.im/Join%20Chat.svg" />
+		</a>
 	</td>
 </tr>
 </table>

asv_bench/benchmarks/groupby.py

@@ -14,7 +14,7 @@
 method_blacklist = {
     'object': {'median', 'prod', 'sem', 'cumsum', 'sum', 'cummin', 'mean',
                'max', 'skew', 'cumprod', 'cummax', 'rank', 'pct_change', 'min',
-               'var', 'mad', 'describe', 'std'},
+               'var', 'mad', 'describe', 'std', 'quantile'},
     'datetime': {'median', 'prod', 'sem', 'cumsum', 'sum', 'mean', 'skew',
                  'cumprod', 'cummax', 'pct_change', 'var', 'mad', 'describe',
                  'std'}
@@ -316,8 +316,9 @@ class GroupByMethods(object):
               ['all', 'any', 'bfill', 'count', 'cumcount', 'cummax', 'cummin',
                'cumprod', 'cumsum', 'describe', 'ffill', 'first', 'head',
                'last', 'mad', 'max', 'min', 'median', 'mean', 'nunique',
-               'pct_change', 'prod', 'rank', 'sem', 'shift', 'size', 'skew',
-               'std', 'sum', 'tail', 'unique', 'value_counts', 'var'],
+               'pct_change', 'prod', 'quantile', 'rank', 'sem', 'shift',
+               'size', 'skew', 'std', 'sum', 'tail', 'unique', 'value_counts',
+               'var'],
               ['direct', 'transformation']]
 
     def setup(self, dtype, method, application):

asv_bench/benchmarks/series_methods.py

@@ -124,6 +124,25 @@ def time_dropna(self, dtype):
         self.s.dropna()
 
 
+class SearchSorted(object):
+
+    goal_time = 0.2
+    params = ['int8', 'int16', 'int32', 'int64',
+              'uint8', 'uint16', 'uint32', 'uint64',
+              'float16', 'float32', 'float64',
+              'str']
+    param_names = ['dtype']
+
+    def setup(self, dtype):
+        N = 10**5
+        data = np.array([1] * N + [2] * N + [3] * N).astype(dtype)
+        self.s = Series(data)
+
+    def time_searchsorted(self, dtype):
+        key = '2' if dtype == 'str' else 2
+        self.s.searchsorted(key)
+
+
 class Map(object):
 
     params = ['dict', 'Series']

ci/code_checks.sh

@@ -241,8 +241,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL06, GL07, GL09, SS04, PR03, PR05, PR10, EX04, RT04, RT05, SS05, SA05)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL06,GL07,GL09,SS04,PR03,PR04,PR05,EX04,RT04,RT05,SS05,SA05
+    MSG='Validate docstrings (GL03, GL06, GL07, GL09, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT04, RT05, SA05)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL03,GL06,GL07,GL09,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT04,RT05,SA05
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/azure-27-compat.yaml

@@ -18,9 +18,10 @@ dependencies:
   - xlsxwriter=0.5.2
   - xlwt=0.7.5
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
+  - isort
   - pip:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1

ci/deps/azure-27-locale.yaml

@@ -20,10 +20,11 @@ dependencies:
   - xlsxwriter=0.5.2
   - xlwt=0.7.5
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0
+  - isort
   - pip:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1

ci/deps/azure-36-locale_slow.yaml

@@ -26,9 +26,10 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - moto
+  - isort
   - pip:
     - hypothesis>=3.58.0

ci/deps/azure-37-locale.yaml

@@ -25,9 +25,10 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
+  - isort
   - pip:
     - hypothesis>=3.58.0
     - moto  # latest moto in conda-forge fails with 3.7, move to conda dependencies when this is fixed

ci/deps/azure-37-numpydev.yaml

@@ -6,10 +6,11 @@ dependencies:
   - pytz
   - Cython>=0.28.2
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0
+  - isort
   - pip:
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"

ci/deps/azure-macos-35.yaml

@@ -21,10 +21,11 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
-  # universal
-  - pytest
-  - pytest-xdist
-  - pytest-mock
+  - isort
   - pip:
     - python-dateutil==2.5.3
+    # universal
+    - pytest>=4.0.2
+    - pytest-xdist
+    - pytest-mock
     - hypothesis>=3.58.0

ci/deps/azure-windows-27.yaml

@@ -25,8 +25,9 @@ dependencies:
   - xlwt
   # universal
   - cython>=0.28.2
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - moto
   - hypothesis>=3.58.0
+  - isort

ci/deps/azure-windows-36.yaml

@@ -23,7 +23,8 @@ dependencies:
   - xlwt
   # universal
   - cython>=0.28.2
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0
+  - isort

ci/deps/travis-27.yaml

@@ -39,11 +39,12 @@ dependencies:
   - xlsxwriter=0.5.2
   - xlwt=0.7.5
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - moto==1.3.4
   - hypothesis>=3.58.0
+  - isort
   - pip:
     - backports.lzma
     - pandas-gbq

ci/deps/travis-36-doc.yaml

@@ -41,5 +41,6 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
+  - isort

ci/deps/travis-36-locale.yaml

@@ -28,9 +28,10 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - moto
+  - isort
   - pip:
     - hypothesis>=3.58.0

ci/deps/travis-36-slow.yaml

@@ -25,8 +25,9 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - moto
   - hypothesis>=3.58.0
+  - isort

ci/deps/travis-36.yaml

@@ -33,11 +33,12 @@ dependencies:
   - xlsxwriter
   - xlwt
   # universal
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-cov
   - pytest-mock
   - hypothesis>=3.58.0
+  - isort
   - pip:
     - brotlipy
     - coverage

ci/deps/travis-37.yaml

@@ -12,10 +12,11 @@ dependencies:
   - nomkl
   - pyarrow
   - pytz
-  - pytest
+  - pytest>=4.0.2
   - pytest-xdist
   - pytest-mock
   - hypothesis>=3.58.0
   - s3fs
+  - isort
   - pip:
     - moto

doc/make.py

@@ -294,14 +294,16 @@ def main():
                            help='number of jobs used by sphinx-build')
     argparser.add_argument('--no-api',
                            default=False,
-                           help='ommit api and autosummary',
+                           help='omit api and autosummary',
                            action='store_true')
     argparser.add_argument('--single',
                            metavar='FILENAME',
                            type=str,
                            default=None,
-                           help=('filename of section or method name to '
-                                 'compile, e.g. "indexing", "DataFrame.join"'))
+                           help=('filename (relative to the "source" folder)'
+                                 ' of section or method name to compile, e.g. '
+                                 '"development/contributing.rst",'
+                                 ' "ecosystem.rst", "pandas.DataFrame.join"'))
     argparser.add_argument('--python-path',
                            type=str,
                            default=os.path.dirname(DOC_PATH),
@@ -323,7 +325,7 @@ def main():
     # the import of `python_path` correctly. The latter is used to resolve
     # the import within the module, injecting it into the global namespace
     os.environ['PYTHONPATH'] = args.python_path
-    sys.path.append(args.python_path)
+    sys.path.insert(0, args.python_path)
     globals()['pandas'] = importlib.import_module('pandas')
 
     # Set the matplotlib backend to the non-interactive Agg backend for all

doc/source/conf.py

@@ -98,9 +98,9 @@
                 if (fname == 'index.rst'
                         and os.path.abspath(dirname) == source_path):
                     continue
-                elif pattern == '-api' and dirname == 'api':
+                elif pattern == '-api' and dirname == 'reference':
                     exclude_patterns.append(fname)
-                elif fname != pattern:
+                elif pattern != '-api' and fname != pattern:
                     exclude_patterns.append(fname)
 
 with open(os.path.join(source_path, 'index.rst.template')) as f:

doc/source/development/contributing.rst

@@ -54,7 +54,7 @@ Bug reports must:
       ...
       ```
 
-#. Include the full version string of *pandas* and its dependencies. You can use the built in function::
+#. Include the full version string of *pandas* and its dependencies. You can use the built-in function::
 
       >>> import pandas as pd
       >>> pd.show_versions()
@@ -178,6 +178,7 @@ We'll now kick off a three-step process:
    # Create and activate the build environment
    conda env create -f environment.yml
    conda activate pandas-dev
+   conda uninstall --force pandas
 
    # or with older versions of Anaconda:
    source activate pandas-dev
@@ -211,7 +212,7 @@ See the full conda docs `here <http://conda.pydata.org/docs>`__.
 Creating a Python Environment (pip)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-If you aren't using conda for you development environment, follow these instructions.
+If you aren't using conda for your development environment, follow these instructions.
 You'll need to have at least python3.5 installed on your system.
 
 .. code-block:: none
@@ -428,14 +429,14 @@ reducing the turn-around time for checking your changes.
     python make.py clean
     python make.py --no-api
 
-    # compile the docs with only a single
-    # section, that which is in indexing.rst
+    # compile the docs with only a single section, relative to the "source" folder.
+    # For example, compiling only this guide (docs/source/development/contributing.rst)
     python make.py clean
-    python make.py --single indexing
+    python make.py --single development/contributing.rst
 
     # compile the reference docs for a single function
     python make.py clean
-    python make.py --single DataFrame.join
+    python make.py --single pandas.DataFrame.join
 
 For comparison, a full documentation build may take 15 minutes, but a single
 section may take 15 seconds. Subsequent builds, which only process portions
@@ -484,7 +485,7 @@ contributing them to the project::
 
    ./ci/code_checks.sh
 
-The script verify the linting of code files, it looks for common mistake patterns
+The script verifies the linting of code files, it looks for common mistake patterns
 (like missing spaces around sphinx directives that make the documentation not
 being rendered properly) and it also validates the doctests. It is possible to
 run the checks independently by using the parameters ``lint``, ``patterns`` and
@@ -675,7 +676,7 @@ Otherwise, you need to do it manually:
 
 You'll also need to
 
-1. write a new test that asserts a warning is issued when calling with the deprecated argument
+1. Write a new test that asserts a warning is issued when calling with the deprecated argument
 2. Update all of pandas existing tests and code to use the new argument
 
 See :ref:`contributing.warnings` for more.
@@ -731,7 +732,7 @@ extensions in `numpy.testing
 
 .. note::
 
-   The earliest supported pytest version is 3.6.0.
+   The earliest supported pytest version is 4.0.2.
 
 Writing tests
 ~~~~~~~~~~~~~

doc/source/development/extending.rst

@@ -33,8 +33,9 @@ decorate a class, providing the name of attribute to add. The class's
 
        @staticmethod
        def _validate(obj):
-           if 'lat' not in obj.columns or 'lon' not in obj.columns:
-               raise AttributeError("Must have 'lat' and 'lon'.")
+           # verify there is a column latitude and a column longitude
+           if 'latitude' not in obj.columns or 'longitude' not in obj.columns:
+               raise AttributeError("Must have 'latitude' and 'longitude'.")
 
        @property
        def center(self):

doc/source/development/index.rst

@@ -6,6 +6,9 @@
 Development
 ===========
 
+.. If you update this toctree, also update the manual toctree in the
+   main index.rst.template
+
 .. toctree::
     :maxdepth: 2
 

doc/source/getting_started/10min.rst

@@ -103,7 +103,7 @@ Display the index, columns:
    df.columns
 
 :meth:`DataFrame.to_numpy` gives a NumPy representation of the underlying data.
-Note that his can be an expensive operation when your :class:`DataFrame` has
+Note that this can be an expensive operation when your :class:`DataFrame` has
 columns with different data types, which comes down to a fundamental difference
 between pandas and NumPy: **NumPy arrays have one dtype for the entire array,
 while pandas DataFrames have one dtype per column**. When you call