Merge remote-tracking branch 'upstream/master' into TYP-core-nanops

Author: MomIsBestFriend

.pre-commit-config.yaml

@@ -20,11 +20,11 @@ repos:
     rev: v0.730
     hooks:
      -  id: mypy
-        # We run mypy over all files because of:
-        #  * changes in type definitions may affect non-touched files.
-        #  * Running it with `mypy pandas` and the filenames will lead to
-        #    spurious duplicate module errors,
-        #    see also https://github.com/pre-commit/mirrors-mypy/issues/5
-        pass_filenames: false
         args:
-        - pandas
+          # As long as a some files are excluded from check-untyped-defs
+          # we have to exclude it from the pre-commit hook as the configuration
+          # is based on modules but the hook runs on files.
+          - --no-check-untyped-defs
+          - --follow-imports
+          - skip
+        files: pandas/

.travis.yml

@@ -25,25 +25,22 @@ git:
 
 matrix:
     fast_finish: true
-    exclude:
-      # Exclude the default Python 3.5 build
-      - python: 3.5
 
     include:
     - env:
-        - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network)"
+        - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
-        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
+        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
       services:
         - mysql
         - postgresql
 
     - env:
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
+        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
       services:
         - mysql
         - postgresql

asv_bench/asv.conf.json

@@ -43,6 +43,7 @@
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],
+        "numba": [],
         "numexpr": [],
         "pytables": [null, ""],  // platform dependent, see excludes below
         "tables": [null, ""],

asv_bench/benchmarks/rolling.py

@@ -44,6 +44,27 @@ def time_rolling(self, constructor, window, dtype, function, raw):
         self.roll.apply(function, raw=raw)
 
 
+class Engine:
+    params = (
+        ["DataFrame", "Series"],
+        ["int", "float"],
+        [np.sum, lambda x: np.sum(x) + 5],
+        ["cython", "numba"],
+    )
+    param_names = ["constructor", "dtype", "function", "engine"]
+
+    def setup(self, constructor, dtype, function, engine):
+        N = 10 ** 3
+        arr = (100 * np.random.random(N)).astype(dtype)
+        self.data = getattr(pd, constructor)(arr)
+
+    def time_rolling_apply(self, constructor, dtype, function, engine):
+        self.data.rolling(10).apply(function, raw=True, engine=engine)
+
+    def time_expanding_apply(self, constructor, dtype, function, engine):
+        self.data.expanding().apply(function, raw=True, engine=engine)
+
+
 class ExpandingMethods:
 
     params = (

ci/azure/posix.yml

@@ -18,7 +18,7 @@ jobs:
         py36_minimum_versions:
           ENV_FILE: ci/deps/azure-36-minimum_versions.yaml
           CONDA_PY: "36"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not clipboard"
 
         py36_locale_slow_old_np:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
@@ -36,12 +36,12 @@ jobs:
           PATTERN: "not slow and not network"
           LANG: "it_IT.utf8"
           LC_ALL: "it_IT.utf8"
-          EXTRA_APT: "language-pack-it"
+          EXTRA_APT: "language-pack-it xsel"
 
         py36_32bit:
           ENV_FILE: ci/deps/azure-36-32bit.yaml
           CONDA_PY: "36"
-          PATTERN: "not slow and not network"
+          PATTERN: "not slow and not network and not clipboard"
           BITS32: "yes"
 
         py37_locale:
@@ -50,7 +50,7 @@ jobs:
           PATTERN: "not slow and not network"
           LANG: "zh_CN.utf8"
           LC_ALL: "zh_CN.utf8"
-          EXTRA_APT: "language-pack-zh-hans"
+          EXTRA_APT: "language-pack-zh-hans xsel"
 
         py37_np_dev:
           ENV_FILE: ci/deps/azure-37-numpydev.yaml

ci/code_checks.sh

@@ -314,8 +314,8 @@ fi
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT01, RT04, RT05, SA01, SA02, SA03, SA05)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT01,RT04,RT05,SA01,SA02,SA03,SA05
+    MSG='Validate docstrings (GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT01, RT04, RT05, SA02, SA03, SA05)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT01,RT04,RT05,SA02,SA03,SA05
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/run_tests.sh

@@ -14,14 +14,14 @@ if [ "$COVERAGE" ]; then
     COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME"
 fi
 
-PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
-
-# Travis does not have have an X server
-if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-    DISPLAY=DISPLAY=:99.0
-    PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD"
+# If no X server is found, we use xvfb to emulate it
+if [[ $(uname) == "Linux" && -z $DISPLAY ]]; then
+    export DISPLAY=":0"
+    XVFB="xvfb-run "
 fi
 
+PYTEST_CMD="${XVFB}pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
+
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"
 

ci/setup_env.sh

@@ -114,6 +114,11 @@ echo "remove postgres if has been installed with conda"
 echo "we use the one from the CI"
 conda remove postgresql -y --force || true
 
+echo
+echo "remove qt"
+echo "causes problems with the clipboard, we use xsel for that"
+conda remove qt -y --force || true
+
 echo
 echo "conda list pandas"
 conda list pandas

doc/source/conf.py

@@ -10,6 +10,7 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+from datetime import datetime
 import importlib
 import inspect
 import logging
@@ -137,7 +138,7 @@
 
 # General information about the project.
 project = "pandas"
-copyright = "2008-2014, the pandas development team"
+copyright = f"2008-{datetime.now().year}, the pandas development team"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

doc/source/development/contributing.rst

@@ -354,9 +354,9 @@ About the *pandas* documentation
 --------------------------------
 
 The documentation is written in **reStructuredText**, which is almost like writing
-in plain English, and built using `Sphinx <http://sphinx.pocoo.org/>`__. The
+in plain English, and built using `Sphinx <http://www.sphinx-doc.org/en/master/>`__. The
 Sphinx Documentation has an excellent `introduction to reST
-<http://sphinx.pocoo.org/rest.html>`__. Review the Sphinx docs to perform more
+<https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html>`__. Review the Sphinx docs to perform more
 complex changes to the documentation as well.
 
 Some other important things to know about the docs:

doc/source/ecosystem.rst

@@ -41,6 +41,16 @@ Pyjanitor provides a clean API for cleaning data, using method chaining.
 Engarde is a lightweight library used to explicitly state assumptions about your datasets
 and check that they're *actually* true.
 
+`pandas-path <https://github.com/drivendataorg/pandas-path/>`__
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since Python 3.4, `pathlib <https://docs.python.org/3/library/pathlib.html>`_ has been
+included in the Python standard library. Path objects provide a simple
+and delightful way to interact with the file system. The pandas-path package enables the
+Path API for pandas through a custom accessor ``.path``. Getting just the filenames from
+a series of full file paths is as simple as ``my_files.path.name``. Other convenient operations like
+joining paths, replacing file extensions, and checking if files exist are also available.
+
 .. _ecosystem.stats:
 
 Statistics and machine learning
@@ -386,12 +396,16 @@ A directory of projects providing
 :ref:`extension accessors <extending.register-accessors>`. This is for users to
 discover new accessors and for library authors to coordinate on the namespace.
 
-============== ========== =========================
-Library        Accessor   Classes
-============== ========== =========================
-`cyberpandas`_ ``ip``     ``Series``
-`pdvega`_      ``vgplot`` ``Series``, ``DataFrame``
-============== ========== =========================
+=============== ========== ========================= ===============================================================
+Library         Accessor   Classes                   Description
+=============== ========== ========================= ===============================================================
+`cyberpandas`_  ``ip``     ``Series``                Provides common operations for working with IP addresses.
+`pdvega`_       ``vgplot`` ``Series``, ``DataFrame`` Provides plotting functions from the Altair_ library.
+`pandas_path`_  ``path``   ``Index``, ``Series``     Provides `pathlib.Path`_ functions for Series.
+=============== ========== ========================= ===============================================================
 
 .. _cyberpandas: https://cyberpandas.readthedocs.io/en/latest
 .. _pdvega: https://altair-viz.github.io/pdvega/
+.. _Altair: https://altair-viz.github.io/
+.. _pandas_path: https://github.com/drivendataorg/pandas-path/
+.. _pathlib.Path: https://docs.python.org/3/library/pathlib.html

doc/source/getting_started/dsintro.rst

@@ -136,7 +136,7 @@ Like a NumPy array, a pandas Series has a :attr:`~Series.dtype`.
 
 This is often a NumPy dtype. However, pandas and 3rd-party libraries
 extend NumPy's type system in a few places, in which case the dtype would
-be a :class:`~pandas.api.extensions.ExtensionDtype`. Some examples within
+be an :class:`~pandas.api.extensions.ExtensionDtype`. Some examples within
 pandas are :ref:`categorical` and :ref:`integer_na`. See :ref:`basics.dtypes`
 for more.
 

doc/source/user_guide/computation.rst

@@ -348,6 +348,7 @@ Numba will be applied in potentially two routines:
 
 1. If ``func`` is a standard Python function, the engine will `JIT <http://numba.pydata.org/numba-doc/latest/user/overview.html>`__
 the passed function. ``func`` can also be a JITed function in which case the engine will not JIT the function again.
+
 2. The engine will JIT the for loop where the apply function is applied to each window.
 
 The ``engine_kwargs`` argument is a dictionary of keyword arguments that will be passed into the

doc/source/user_guide/missing_data.rst

@@ -825,13 +825,10 @@ For example, ``pd.NA`` propagates in arithmetic operations, similarly to
 There are a few special cases when the result is known, even when one of the
 operands is ``NA``.
 
+.. ipython:: python
 
-================ ======
-Operation        Result
-================ ======
-``pd.NA ** 0``   0
-``1 ** pd.NA``   1
-================ ======
+   pd.NA ** 0
+   1 ** pd.NA
 
 In equality and comparison operations, ``pd.NA`` also propagates. This deviates
 from the behaviour of ``np.nan``, where comparisons with ``np.nan`` always

doc/source/whatsnew/v1.0.0.rst

@@ -159,14 +159,14 @@ You can use the alias ``"boolean"`` as well.
 
 .. _whatsnew_100.numba_rolling_apply:
 
-Using Numba in ``rolling.apply``
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Using Numba in ``rolling.apply`` and ``expanding.apply``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-We've added an ``engine`` keyword to :meth:`~core.window.rolling.Rolling.apply` that allows the user to execute the
-routine using `Numba <https://numba.pydata.org/>`__ instead of Cython. Using the Numba engine
-can yield significant performance gains if the apply function can operate on numpy arrays and
+We've added an ``engine`` keyword to :meth:`~core.window.rolling.Rolling.apply` and :meth:`~core.window.expanding.Expanding.apply`
+that allows the user to execute the routine using `Numba <https://numba.pydata.org/>`__ instead of Cython.
+Using the Numba engine can yield significant performance gains if the apply function can operate on numpy arrays and
 the data set is larger (1 million rows or greater). For more details, see
-:ref:`rolling apply documentation <stats.rolling_apply>` (:issue:`28987`)
+:ref:`rolling apply documentation <stats.rolling_apply>` (:issue:`28987`, :issue:`30936`)
 
 .. _whatsnew_100.custom_window:
 
@@ -223,7 +223,7 @@ Other enhancements
 - :meth:`DataFrame.sort_values` and :meth:`Series.sort_values` have gained ``ignore_index`` keyword to be able to reset index after sorting (:issue:`30114`)
 - :meth:`DataFrame.sort_index` and :meth:`Series.sort_index` have gained ``ignore_index`` keyword to reset index (:issue:`30114`)
 - :meth:`DataFrame.drop_duplicates` has gained ``ignore_index`` keyword to reset index (:issue:`30114`)
-- Added new writer for exporting Stata dta files in version 118, ``StataWriter118``.  This format supports exporting strings containing Unicode characters (:issue:`23573`)
+- Added new writer for exporting Stata dta files in versions 118 and 119, ``StataWriterUTF8``.  These files formats support exporting strings containing Unicode characters. Format 119 supports data sets with more than 32,767 variables (:issue:`23573`, :issue:`30959`)
 - :meth:`Series.map` now accepts ``collections.abc.Mapping`` subclasses as a mapper (:issue:`29733`)
 - Added an experimental :attr:`~DataFrame.attrs` for storing global metadata about a dataset (:issue:`29062`)
 - :meth:`Timestamp.fromisocalendar` is now compatible with python 3.8 and above (:issue:`28115`)

doc/source/whatsnew/v1.1.0.rst

@@ -59,6 +59,7 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
+- Bug in :class:`Timestamp` where constructing :class:`Timestamp` from ambiguous epoch time and calling constructor again changed :meth:`Timestamp.value` property (:issue:`24329`)
 -
 -
 
@@ -140,7 +141,7 @@ Reshaping
 ^^^^^^^^^
 
 -
--
+- Bug in :meth:`DataFrame.pivot_table` when only MultiIndexed columns is set (:issue:`17038`)
 
 Sparse
 ^^^^^^

environment.yml

@@ -27,7 +27,6 @@ dependencies:
   # documentation
   - gitpython  # obtain contributors from git for whatsnew
   - sphinx
-  - numpydoc>=0.9.0
 
   # documentation (jupyter notebooks)
   - nbconvert>=5.4.1
@@ -105,3 +104,4 @@ dependencies:
   - tabulate>=0.8.3  # DataFrame.to_markdown
   - pip:
     - git+https://github.com/pandas-dev/pandas-sphinx-theme.git@master
+    - git+https://github.com/numpy/numpydoc