Merge branch 'master' of https://github.com/pandas-dev/pandas into cln-pytables8

Author: jbrockmendel

ci/code_checks.sh

@@ -194,6 +194,10 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' .
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Check for use of comment-based annotation syntax' ; echo $MSG
+    invgrep -R --include="*.py" -P '# type: (?!ignore)' pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Check that no file in the repo contains trailing whitespaces' ; echo $MSG
     set -o pipefail
     if [[ "$AZURE" == "true" ]]; then

ci/deps/azure-36-32bit.yaml

@@ -8,7 +8,6 @@ dependencies:
   # tools
   ### Cython 0.29.13 and pytest 5.0.1 for 32 bits are not available with conda, installing below with pip instead
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/azure-36-locale.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/azure-36-locale_slow.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/azure-36-minimum_versions.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython=0.29.13
   - pytest=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/azure-37-locale.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/azure-37-numpydev.yaml

@@ -8,7 +8,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/azure-macos-36.yaml

@@ -8,7 +8,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/azure-windows-36.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/azure-windows-37.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
   - pytest-azurepipelines
 

ci/deps/travis-36-cov.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
   - pytest-cov  # this is only needed in the coverage build
 

ci/deps/travis-36-locale.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
 
   # pandas dependencies

ci/deps/travis-36-slow.yaml

@@ -9,7 +9,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
 
   # pandas dependencies

ci/deps/travis-37.yaml

@@ -10,7 +10,6 @@ dependencies:
   - cython>=0.29.13
   - pytest>=5.0.1
   - pytest-xdist>=1.21
-  - pytest-mock
   - hypothesis>=3.58.0
 
   # pandas dependencies

ci/deps/travis-38.yaml

@@ -8,7 +8,7 @@ dependencies:
   # tools
   - cython>=0.29.13
   - pytest>=5.0.1
-  - pytest-xdist>=1.29.0  # The rest of the builds use >=1.21, and use pytest-mock
+  - pytest-xdist>=1.21
   - hypothesis>=3.58.0
 
   # pandas dependencies

doc/source/development/contributing.rst

@@ -804,27 +804,27 @@ Types imports should follow the ``from typing import ...`` convention. So rather
 
    import typing
 
-   primes = []  # type: typing.List[int]
+   primes: typing.List[int] = []
 
 You should write
 
 .. code-block:: python
 
    from typing import List, Optional, Union
 
-   primes = []  # type: List[int]
+   primes: List[int] = []
 
 ``Optional`` should be used where applicable, so instead of
 
 .. code-block:: python
 
-   maybe_primes = []  # type: List[Union[int, None]]
+   maybe_primes: List[Union[int, None]] = []
 
 You should write
 
 .. code-block:: python
 
-   maybe_primes = []  # type: List[Optional[int]]
+   maybe_primes: List[Optional[int]] = []
 
 In some cases in the code base classes may define class variables that shadow builtins. This causes an issue as described in `Mypy 1775 <https://github.com/python/mypy/issues/1775#issuecomment-310969854>`_. The defensive solution here is to create an unambiguous alias of the builtin and use that without your annotation. For example, if you come across a definition like
 
@@ -840,7 +840,7 @@ The appropriate way to annotate this would be as follows
    str_type = str
 
    class SomeClass2:
-       str = None  # type: str_type
+       str: str_type = None
 
 In some cases you may be tempted to use ``cast`` from the typing module when you know better than the analyzer. This occurs particularly when using custom inference functions. For example
 
@@ -946,7 +946,7 @@ extensions in `numpy.testing
 
 .. note::
 
-   The earliest supported pytest version is 4.0.2.
+   The earliest supported pytest version is 5.0.1.
 
 Writing tests
 ~~~~~~~~~~~~~

doc/source/getting_started/install.rst

@@ -177,7 +177,7 @@ pandas is equipped with an exhaustive set of unit tests, covering about 97% of
 the code base as of this writing. To run it on your machine to verify that
 everything is working (and that you have all of the dependencies, soft and hard,
 installed), make sure you have `pytest
-<http://docs.pytest.org/en/latest/>`__ >= 4.0.2 and `Hypothesis
+<http://docs.pytest.org/en/latest/>`__ >= 5.0.1 and `Hypothesis
 <https://hypothesis.readthedocs.io/>`__ >= 3.58, then run:
 
 ::

doc/source/whatsnew/v1.0.0.rst

@@ -253,6 +253,7 @@ Other API changes
   See :ref:`units registration <whatsnew_1000.matplotlib_units>` for more.
 - :meth:`Series.dropna` has dropped its ``**kwargs`` argument in favor of a single ``how`` parameter.
   Supplying anything else than ``how`` to ``**kwargs`` raised a ``TypeError`` previously (:issue:`29388`)
+- When testing pandas, the new minimum required version of pytest is 5.0.1 (:issue:`29664`)
 -
 
 
@@ -321,6 +322,7 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
 - :meth:`pandas.Series.str.cat` now defaults to aligning ``others``, using ``join='left'`` (:issue:`27611`)
 - :meth:`pandas.Series.str.cat` does not accept list-likes *within* list-likes anymore (:issue:`27611`)
 - Removed the previously deprecated :meth:`ExtensionArray._formatting_values`. Use :attr:`ExtensionArray._formatter` instead. (:issue:`23601`)
+- Removed support for legacy HDF5 formats (:issue:`29787`)
 - :func:`read_excel` removed support for "skip_footer" argument, use "skipfooter" instead (:issue:`18836`)
 - :meth:`DataFrame.to_records` no longer supports the argument "convert_datetime64" (:issue:`18902`)
 - Removed the previously deprecated ``IntervalIndex.from_intervals`` in favor of the :class:`IntervalIndex` constructor (:issue:`19263`)
@@ -487,6 +489,7 @@ I/O
 - Bug in :meth:`Styler.background_gradient` not able to work with dtype ``Int64`` (:issue:`28869`)
 - Bug in :meth:`DataFrame.to_clipboard` which did not work reliably in ipython (:issue:`22707`)
 - Bug in :func:`read_json` where default encoding was not set to ``utf-8`` (:issue:`29565`)
+- Bug in :class:`PythonParser` where str and bytes were being mixed when dealing with the decimal field (:issue:`29650`)
 -
 
 Plotting

pandas/_config/config.py

@@ -58,16 +58,16 @@
 RegisteredOption = namedtuple("RegisteredOption", "key defval doc validator cb")
 
 # holds deprecated option metdata
-_deprecated_options = {}  # type: Dict[str, DeprecatedOption]
+_deprecated_options: Dict[str, DeprecatedOption] = {}
 
 # holds registered option metdata
-_registered_options = {}  # type: Dict[str, RegisteredOption]
+_registered_options: Dict[str, RegisteredOption] = {}
 
 # holds the current values for registered options
-_global_config = {}  # type: Dict[str, str]
+_global_config: Dict[str, str] = {}
 
 # keys which have a special meaning
-_reserved_keys = ["all"]  # type: List[str]
+_reserved_keys: List[str] = ["all"]
 
 
 class OptionError(AttributeError, KeyError):

pandas/_libs/parsers.pyx

@@ -1406,59 +1406,6 @@ cdef inline StringPath _string_path(char *encoding):
 # Type conversions / inference support code
 
 
-cdef _string_box_factorize(parser_t *parser, int64_t col,
-                           int64_t line_start, int64_t line_end,
-                           bint na_filter, kh_str_starts_t *na_hashset):
-    cdef:
-        int error, na_count = 0
-        Py_ssize_t i, lines
-        coliter_t it
-        const char *word = NULL
-        ndarray[object] result
-
-        int ret = 0
-        kh_strbox_t *table
-
-        object pyval
-
-        object NA = na_values[np.object_]
-        khiter_t k
-
-    table = kh_init_strbox()
-    lines = line_end - line_start
-    result = np.empty(lines, dtype=np.object_)
-    coliter_setup(&it, parser, col, line_start)
-
-    for i in range(lines):
-        COLITER_NEXT(it, word)
-
-        if na_filter:
-            if kh_get_str_starts_item(na_hashset, word):
-                # in the hash table
-                na_count += 1
-                result[i] = NA
-                continue
-
-        k = kh_get_strbox(table, word)
-
-        # in the hash table
-        if k != table.n_buckets:
-            # this increments the refcount, but need to test
-            pyval = <object>table.vals[k]
-        else:
-            # box it. new ref?
-            pyval = PyBytes_FromString(word)
-
-            k = kh_put_strbox(table, word, &ret)
-            table.vals[k] = <PyObject*>pyval
-
-        result[i] = pyval
-
-    kh_destroy_strbox(table)
-
-    return result, na_count
-
-
 cdef _string_box_utf8(parser_t *parser, int64_t col,
                       int64_t line_start, int64_t line_end,
                       bint na_filter, kh_str_starts_t *na_hashset):

pandas/_libs/src/ujson/python/objToJSON.c

@@ -399,22 +399,6 @@ static void *CLong(JSOBJ obj, JSONTypeContext *tc, void *outValue,
     return NULL;
 }
 
-#ifdef _LP64
-static void *PyIntToINT64(JSOBJ _obj, JSONTypeContext *tc, void *outValue,
-                          size_t *_outLen) {
-    PyObject *obj = (PyObject *)_obj;
-    *((JSINT64 *)outValue) = PyLong_AsLong(obj);
-    return NULL;
-}
-#else
-static void *PyIntToINT32(JSOBJ _obj, JSONTypeContext *tc, void *outValue,
-                          size_t *_outLen) {
-    PyObject *obj = (PyObject *)_obj;
-    *((JSINT32 *)outValue) = PyLong_AsLong(obj);
-    return NULL;
-}
-#endif
-
 static void *PyLongToINT64(JSOBJ _obj, JSONTypeContext *tc, void *outValue,
                            size_t *_outLen) {
     *((JSINT64 *)outValue) = GET_TC(tc)->longValue;

pandas/_libs/tslibs/c_timestamp.pyx

@@ -55,11 +55,11 @@ def maybe_integer_op_deprecated(obj):
     # GH#22535 add/sub of integers and int-arrays is deprecated
     if obj.freq is not None:
         warnings.warn("Addition/subtraction of integers and integer-arrays "
-                      "to {cls} is deprecated, will be removed in a future "
+                      f"to {type(obj).__name__} is deprecated, "
+                      "will be removed in a future "
                       "version.  Instead of adding/subtracting `n`, use "
                       "`n * self.freq`"
-                      .format(cls=type(obj).__name__),
-                      FutureWarning)
+                      , FutureWarning)
 
 
 cdef class _Timestamp(datetime):
@@ -144,11 +144,10 @@ cdef class _Timestamp(datetime):
             # e.g. tzlocal has no `strftime`
             pass
 
-        tz = ", tz='{0}'".format(zone) if zone is not None else ""
-        freq = "" if self.freq is None else ", freq='{0}'".format(self.freqstr)
+        tz = f", tz='{zone}'" if zone is not None else ""
+        freq = "" if self.freq is None else f", freq='{self.freqstr}'"
 
-        return "Timestamp('{stamp}'{tz}{freq})".format(stamp=stamp,
-                                                       tz=tz, freq=freq)
+        return f"Timestamp('{stamp}'{tz}{freq})"
 
     cdef bint _compare_outside_nanorange(_Timestamp self, datetime other,
                                          int op) except -1:
@@ -370,23 +369,22 @@ cdef class _Timestamp(datetime):
 
     @property
     def _repr_base(self) -> str:
-        return '{date} {time}'.format(date=self._date_repr,
-                                      time=self._time_repr)
+        return f"{self._date_repr} {self._time_repr}"
 
     @property
     def _date_repr(self) -> str:
         # Ideal here would be self.strftime("%Y-%m-%d"), but
         # the datetime strftime() methods require year >= 1900
-        return '%d-%.2d-%.2d' % (self.year, self.month, self.day)
+        return f'{self.year}-{self.month:02d}-{self.day:02d}'
 
     @property
     def _time_repr(self) -> str:
-        result = '%.2d:%.2d:%.2d' % (self.hour, self.minute, self.second)
+        result = f'{self.hour:02d}:{self.minute:02d}:{self.second:02d}'
 
         if self.nanosecond != 0:
-            result += '.%.9d' % (self.nanosecond + 1000 * self.microsecond)
+            result += f'.{self.nanosecond + 1000 * self.microsecond:09d}'
         elif self.microsecond != 0:
-            result += '.%.6d' % self.microsecond
+            result += f'.{self.microsecond:06d}'
 
         return result
 

pandas/_libs/tslibs/conversion.pyx

@@ -197,7 +197,7 @@ def datetime_to_datetime64(object[:] values):
                 iresult[i] = pydatetime_to_dt64(val, &dts)
                 check_dts_bounds(&dts)
         else:
-            raise TypeError('Unrecognized value type: %s' % type(val))
+            raise TypeError(f'Unrecognized value type: {type(val)}')
 
     return result, inferred_tz
 
@@ -326,8 +326,8 @@ cdef convert_to_tsobject(object ts, object tz, object unit,
         raise ValueError("Cannot convert Period to Timestamp "
                          "unambiguously. Use to_timestamp")
     else:
-        raise TypeError('Cannot convert input [{}] of type {} to '
-                        'Timestamp'.format(ts, type(ts)))
+        raise TypeError(f'Cannot convert input [{ts}] of type {type(ts)} to '
+                        f'Timestamp')
 
     if tz is not None:
         localize_tso(obj, tz)
@@ -686,7 +686,7 @@ def normalize_date(dt: object) -> datetime:
     elif PyDate_Check(dt):
         return datetime(dt.year, dt.month, dt.day)
     else:
-        raise TypeError('Unrecognized type: %s' % type(dt))
+        raise TypeError(f'Unrecognized type: {type(dt)}')
 
 
 @cython.wraparound(False)