Merge branch 'master' into feature/groupby-repr-ellipses-1135

Author: benjaminarjun

doc/source/whatsnew/v0.25.0.rst

@@ -354,7 +354,7 @@ I/O
 - Bug in :meth:`DataFrame.to_string` and :meth:`DataFrame.to_latex` that would lead to incorrect output when the ``header`` keyword is used (:issue:`16718`)
 - Bug in :func:`read_csv` not properly interpreting the UTF8 encoded filenames on Windows on Python 3.6+ (:issue:`15086`)
 - Improved performance in :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` when converting columns that have missing values (:issue:`25772`)
-
+- Bug in ``read_csv`` which would not raise ``ValueError`` if a column index in ``usecols`` was out of bounds (:issue:`25623`)
 
 Plotting
 ^^^^^^^^
@@ -373,6 +373,7 @@ Groupby/Resample/Rolling
 - Bug in :func:`pandas.core.groupby.GroupBy.agg` when applying a aggregation function to timezone aware data (:issue:`23683`)
 - Bug in :func:`pandas.core.groupby.GroupBy.first` and :func:`pandas.core.groupby.GroupBy.last` where timezone information would be dropped (:issue:`21603`)
 - Ensured that ordering of outputs in ``groupby`` aggregation functions is consistent across all versions of Python (:issue:`25692`)
+- Bug in :func:`idxmax` and :func:`idxmin` on :meth:`DataFrame.groupby` with datetime column would return incorrect dtype (:issue:`25444`, :issue:`15306`)
 
 
 Reshaping
@@ -396,7 +397,7 @@ Sparse
 Other
 ^^^^^
 
--
+- Bug in :class:`Series` and :class:`DataFrame` repr where ``np.datetime64('NaT')`` and ``np.timedelta64('NaT')`` with ``dtype=object`` would be represented as ``NaN`` (:issue:`25445`)
 -
 -
 

pandas/_libs/hashtable_class_helper.pxi.in

@@ -9,6 +9,8 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 # VectorData
 # ----------------------------------------------------------------------
 
+from pandas._libs.tslibs.util cimport get_c_string
+
 {{py:
 
 # name, dtype, arg
@@ -595,7 +597,7 @@ cdef class StringHashTable(HashTable):
         cdef:
             khiter_t k
             const char *v
-        v = util.get_c_string(val)
+        v = get_c_string(val)
 
         k = kh_get_str(self.table, v)
         if k != self.table.n_buckets:
@@ -609,7 +611,7 @@ cdef class StringHashTable(HashTable):
             int ret = 0
             const char *v
 
-        v = util.get_c_string(val)
+        v = get_c_string(val)
 
         k = kh_put_str(self.table, v, &ret)
         self.table.keys[k] = key
@@ -632,7 +634,7 @@ cdef class StringHashTable(HashTable):
         vecs = <const char **>malloc(n * sizeof(char *))
         for i in range(n):
             val = values[i]
-            v = util.get_c_string(val)
+            v = get_c_string(val)
             vecs[i] = v
 
         with nogil:
@@ -662,9 +664,9 @@ cdef class StringHashTable(HashTable):
             val = values[i]
 
             if isinstance(val, (str, unicode)):
-                v = util.get_c_string(val)
+                v = get_c_string(val)
             else:
-                v = util.get_c_string(self.na_string_sentinel)
+                v = get_c_string(self.na_string_sentinel)
             vecs[i] = v
 
         with nogil:
@@ -695,9 +697,9 @@ cdef class StringHashTable(HashTable):
             val = values[i]
 
             if isinstance(val, (str, unicode)):
-                v = util.get_c_string(val)
+                v = get_c_string(val)
             else:
-                v = util.get_c_string(self.na_string_sentinel)
+                v = get_c_string(self.na_string_sentinel)
             vecs[i] = v
 
         with nogil:
@@ -776,7 +778,7 @@ cdef class StringHashTable(HashTable):
                 labels[i] = na_sentinel
             else:
                 # if ignore_na is False, we also stringify NaN/None/etc.
-                v = util.get_c_string(val)
+                v = get_c_string(val)
                 vecs[i] = v
 
         # compute

pandas/_libs/parsers.pyx

@@ -592,8 +592,7 @@ cdef class TextReader:
         if not QUOTE_MINIMAL <= quoting <= QUOTE_NONE:
             raise TypeError('bad "quoting" value')
 
-        if not isinstance(quote_char, (str, compat.text_type,
-                                       bytes)) and quote_char is not None:
+        if not isinstance(quote_char, (str, bytes)) and quote_char is not None:
             dtype = type(quote_char).__name__
             raise TypeError('"quotechar" must be string, '
                             'not {dtype}'.format(dtype=dtype))
@@ -2123,7 +2122,7 @@ cdef raise_parser_error(object base, parser_t *parser):
 
             # PyErr_Fetch only returned the error message in *value,
             # so the Exception class must be extracted from *type.
-            if isinstance(old_exc, compat.string_types):
+            if isinstance(old_exc, str):
                 if type != NULL:
                     exc_type = <object>type
                 else:

pandas/_libs/testing.pyx

@@ -1,6 +1,5 @@
 import numpy as np
 
-from pandas import compat
 from pandas.core.dtypes.missing import isna, array_equivalent
 from pandas.core.dtypes.common import is_dtype_equal
 
@@ -108,8 +107,7 @@ cpdef assert_almost_equal(a, b,
     if isinstance(a, dict) or isinstance(b, dict):
         return assert_dict_equal(a, b)
 
-    if (isinstance(a, compat.string_types) or
-            isinstance(b, compat.string_types)):
+    if isinstance(a, str) or isinstance(b, str):
         assert a == b, "%r != %r" % (a, b)
         return True
 

pandas/_libs/tslibs/nattype.pyx

@@ -229,7 +229,7 @@ cdef class _NaT(datetime):
 
     def total_seconds(self):
         """
-        Total duration of timedelta in seconds (to ns precision)
+        Total duration of timedelta in seconds (to ns precision).
         """
         # GH#10939
         return np.nan

pandas/_libs/tslibs/np_datetime.pyx

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from cpython cimport (Py_EQ, Py_NE, Py_GE, Py_GT, Py_LT, Py_LE,
-                      PyUnicode_AsASCIIString)
+from cpython cimport Py_EQ, Py_NE, Py_GE, Py_GT, Py_LT, Py_LE
 
 from cpython.datetime cimport (datetime, date,
                                PyDateTime_IMPORT,
@@ -13,6 +12,7 @@ from cpython.datetime cimport (datetime, date,
 PyDateTime_IMPORT
 
 from numpy cimport int64_t
+from pandas._libs.tslibs.util cimport get_c_string_buf_and_size
 
 cdef extern from "src/datetime/np_datetime.h":
     int cmp_npy_datetimestruct(npy_datetimestruct *a,
@@ -33,7 +33,7 @@ cdef extern from "src/datetime/np_datetime.h":
     npy_datetimestruct _NS_MIN_DTS, _NS_MAX_DTS
 
 cdef extern from "src/datetime/np_datetime_strings.h":
-    int parse_iso_8601_datetime(char *str, int len,
+    int parse_iso_8601_datetime(const char *str, int len,
                                 npy_datetimestruct *out,
                                 int *out_local, int *out_tzoffset)
 
@@ -174,30 +174,9 @@ cdef inline int64_t pydate_to_dt64(date val, npy_datetimestruct *dts):
 cdef inline int _string_to_dts(object val, npy_datetimestruct* dts,
                                int* out_local, int* out_tzoffset) except? -1:
     cdef:
-        int result
-        char *tmp
+        Py_ssize_t length
+        const char* buf
 
-    if isinstance(val, unicode):
-        val = PyUnicode_AsASCIIString(val)
-
-    tmp = val
-    result = _cstring_to_dts(tmp, len(val), dts, out_local, out_tzoffset)
-
-    if result == -1:
-        raise ValueError('Unable to parse %s' % str(val))
-    return result
-
-
-cdef inline int _cstring_to_dts(char *val, int length,
-                                npy_datetimestruct* dts,
-                                int* out_local, int* out_tzoffset) except? -1:
-    # Note: without this "extra layer" between _string_to_dts
-    # and parse_iso_8601_datetime, calling _string_to_dts raises
-    # `SystemError: <class 'str'> returned a result with an error set`
-    # in Python3
-    cdef:
-        int result
-
-    result = parse_iso_8601_datetime(val, length,
-                                     dts, out_local, out_tzoffset)
-    return result
+    buf = get_c_string_buf_and_size(val, &length)
+    return parse_iso_8601_datetime(buf, length,
+                                   dts, out_local, out_tzoffset)

pandas/_libs/tslibs/parsing.pyx

@@ -12,7 +12,6 @@ from cpython.datetime cimport datetime
 import numpy as np
 
 import six
-from six import binary_type, text_type
 
 # Avoid import from outside _libs
 if sys.version_info.major == 2:
@@ -102,7 +101,7 @@ def parse_time_string(arg, freq=None, dayfirst=None, yearfirst=None):
 
     Parameters
     ----------
-    arg : compat.string_types
+    arg : str
     freq : str or DateOffset, default None
         Helps with interpreting time string if supplied
     dayfirst : bool, default None
@@ -537,13 +536,13 @@ class _timelex(object):
         if six.PY2:
             # In Python 2, we can't duck type properly because unicode has
             # a 'decode' function, and we'd be double-decoding
-            if isinstance(instream, (binary_type, bytearray)):
+            if isinstance(instream, (bytes, bytearray)):
                 instream = instream.decode()
         else:
             if getattr(instream, 'decode', None) is not None:
                 instream = instream.decode()
 
-        if isinstance(instream, text_type):
+        if isinstance(instream, str):
             self.stream = instream
         elif getattr(instream, 'read', None) is None:
             raise TypeError(

pandas/_libs/tslibs/period.pyx

@@ -2390,7 +2390,7 @@ class Period(_Period):
 
     Parameters
     ----------
-    value : Period or compat.string_types, default None
+    value : Period or str, default None
         The time period represented (e.g., '4Q2005')
     freq : str, default None
         One of pandas period strings or corresponding objects

pandas/_libs/tslibs/src/datetime/np_datetime_strings.c

@@ -66,12 +66,13 @@ This file implements string parsing and creation for NumPy datetime.
  *
  * Returns 0 on success, -1 on failure.
  */
-int parse_iso_8601_datetime(char *str, int len,
+int parse_iso_8601_datetime(const char *str, int len,
                             npy_datetimestruct *out,
                             int *out_local, int *out_tzoffset) {
     int year_leap = 0;
     int i, numdigits;
-    char *substr, sublen;
+    const char *substr;
+    int sublen;
 
     /* If year-month-day are separated by a valid separator,
      * months/days without leading zeroes will be parsed
@@ -586,7 +587,8 @@ int get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base) {
  */
 int make_iso_8601_datetime(npy_datetimestruct *dts, char *outstr, int outlen,
                            NPY_DATETIMEUNIT base) {
-    char *substr = outstr, sublen = outlen;
+    char *substr = outstr;
+    int sublen = outlen;
     int tmplen;
 
     /*

pandas/_libs/tslibs/src/datetime/np_datetime_strings.h

@@ -54,7 +54,7 @@ This file implements string parsing and creation for NumPy datetime.
  * Returns 0 on success, -1 on failure.
  */
 int
-parse_iso_8601_datetime(char *str, int len,
+parse_iso_8601_datetime(const char *str, int len,
                         npy_datetimestruct *out,
                         int *out_local,
                         int *out_tzoffset);

pandas/_libs/tslibs/timedeltas.pyx

@@ -815,13 +815,29 @@ cdef class _Timedelta(timedelta):
 
     cpdef timedelta to_pytimedelta(_Timedelta self):
         """
-        return an actual datetime.timedelta object
-        note: we lose nanosecond resolution if any
+        Convert a pandas Timedelta object into a python timedelta object.
+
+        Timedelta objects are internally saved as numpy datetime64[ns] dtype.
+        Use to_pytimedelta() to convert to object dtype.
+
+        Returns
+        -------
+        datetime.timedelta or numpy.array of datetime.timedelta
+
+        See Also
+        --------
+        to_timedelta : Convert argument to Timedelta type.
+
+        Notes
+        -----
+        Any nanosecond resolution will be lost.
         """
         return timedelta(microseconds=int(self.value) / 1000)
 
     def to_timedelta64(self):
-        """ Returns a numpy.timedelta64 object with 'ns' precision """
+        """
+        Return a numpy.timedelta64 object with 'ns' precision.
+        """
         return np.timedelta64(self.value, 'ns')
 
     def to_numpy(self, dtype=None, copy=False):
@@ -846,17 +862,21 @@ cdef class _Timedelta(timedelta):
 
     def total_seconds(self):
         """
-        Total duration of timedelta in seconds (to ns precision)
+        Total duration of timedelta in seconds (to ns precision).
         """
         return self.value / 1e9
 
     def view(self, dtype):
-        """ array view compat """
+        """
+        Array view compatibility.
+        """
         return np.timedelta64(self.value).view(dtype)
 
     @property
     def components(self):
-        """ Return a Components NamedTuple-like """
+        """
+        Return a components namedtuple-like.
+        """
         self._ensure_components()
         # return the named tuple
         return Components(self._d, self._h, self._m, self._s,
@@ -1157,6 +1177,7 @@ class Timedelta(_Timedelta):
     -----
     The ``.value`` attribute is always in ns.
     """
+
     def __new__(cls, object value=_no_input, unit=None, **kwargs):
         cdef _Timedelta td_base
 

pandas/_libs/tslibs/timestamps.pyx

@@ -539,7 +539,8 @@ cdef class _Timestamp(datetime):
 
 
 class Timestamp(_Timestamp):
-    """Pandas replacement for datetime.datetime
+    """
+    Pandas replacement for python datetime.datetime object.
 
     Timestamp is the pandas equivalent of python's Datetime
     and is interchangeable with it in most cases. It's the type used
@@ -549,9 +550,9 @@ class Timestamp(_Timestamp):
     Parameters
     ----------
     ts_input : datetime-like, str, int, float
-        Value to be converted to Timestamp
+        Value to be converted to Timestamp.
     freq : str, DateOffset
-        Offset which Timestamp will have
+        Offset which Timestamp will have.
     tz : str, pytz.timezone, dateutil.tz.tzfile or None
         Time zone for time which Timestamp will have.
     unit : str