Merge remote-tracking branch 'upstream/master' into ea-unstack

Author: TomAugspurger

ci/code_checks.sh

@@ -49,7 +49,7 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     # Note: this grep pattern is (intended to be) equivalent to the python
     # regex r'(?<![ ->])> '
     MSG='Linting .pyx code for spacing conventions in casting' ; echo $MSG
-    ! grep -r -E --include '*.pyx' --include '*.pxi.in' '> ' pandas/_libs | grep -v '[ ->]> '
+    ! grep -r -E --include '*.pyx' --include '*.pxi.in' '[a-zA-Z0-9*]> ' pandas/_libs
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # readability/casting: Warnings about C casting instead of C++ casting

ci/travis-37-numpydev.yaml

@@ -13,5 +13,5 @@ dependencies:
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"
     - "--pre"
-    - "numpy<=1.16.0.dev0+20181015190246"
+    - "numpy"
     - "scipy"

doc/source/whatsnew/v0.24.0.txt

@@ -1047,7 +1047,7 @@ Removal of prior version deprecations/changes
 Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Slicing Series and Dataframes with an monotonically increasing :class:`CategoricalIndex`
+- Slicing Series and DataFrames with an monotonically increasing :class:`CategoricalIndex`
   is now very fast and has speed comparable to slicing with an ``Int64Index``.
   The speed increase is both when indexing by label (using .loc) and position(.iloc) (:issue:`20395`)
   Slicing a monotonically increasing :class:`CategoricalIndex` itself (i.e. ``ci[1000:2000]``)
@@ -1119,6 +1119,7 @@ Datetimelike
 - Bug in :func:`DataFrame.combine` with datetimelike values raising a TypeError (:issue:`23079`)
 - Bug in :func:`date_range` with frequency of ``Day`` or higher where dates sufficiently far in the future could wrap around to the past instead of raising ``OutOfBoundsDatetime`` (:issue:`14187`)
 - Bug in :class:`PeriodIndex` with attribute ``freq.n`` greater than 1 where adding a :class:`DateOffset` object would return incorrect results (:issue:`23215`)
+- Bug in :class:`Series` that interpreted string indices as lists of characters when setting datetimelike values (:issue:`23451`)
 
 Timedelta
 ^^^^^^^^^
@@ -1132,6 +1133,8 @@ Timedelta
 - Fixed bug in adding a :class:`DataFrame` with all-`timedelta64[ns]` dtypes to a :class:`DataFrame` with all-integer dtypes returning incorrect results instead of raising ``TypeError`` (:issue:`22696`)
 - Bug in :class:`TimedeltaIndex` where adding a timezone-aware datetime scalar incorrectly returned a timezone-naive :class:`DatetimeIndex` (:issue:`23215`)
 - Bug in :class:`TimedeltaIndex` where adding ``np.timedelta64('NaT')`` incorrectly returned an all-`NaT` :class:`DatetimeIndex` instead of an all-`NaT` :class:`TimedeltaIndex` (:issue:`23215`)
+- Bug in :class:`Timedelta` and :func:`to_timedelta()` have inconsistencies in supported unit string (:issue:`21762`)
+
 
 Timezones
 ^^^^^^^^^
@@ -1149,7 +1152,7 @@ Timezones
 - Fixed bug where :meth:`DataFrame.describe` and :meth:`Series.describe` on tz-aware datetimes did not show `first` and `last` result (:issue:`21328`)
 - Bug in :class:`DatetimeIndex` comparisons failing to raise ``TypeError`` when comparing timezone-aware ``DatetimeIndex`` against ``np.datetime64`` (:issue:`22074`)
 - Bug in ``DataFrame`` assignment with a timezone-aware scalar (:issue:`19843`)
-- Bug in :func:`Dataframe.asof` that raised a ``TypeError`` when attempting to compare tz-naive and tz-aware timestamps (:issue:`21194`)
+- Bug in :func:`DataFrame.asof` that raised a ``TypeError`` when attempting to compare tz-naive and tz-aware timestamps (:issue:`21194`)
 - Bug when constructing a :class:`DatetimeIndex` with :class:`Timestamp`s constructed with the ``replace`` method across DST (:issue:`18785`)
 - Bug when setting a new value with :meth:`DataFrame.loc` with a :class:`DatetimeIndex` with a DST transition (:issue:`18308`, :issue:`20724`)
 - Bug in :meth:`DatetimeIndex.unique` that did not re-localize tz-aware dates correctly (:issue:`21737`)
@@ -1282,6 +1285,7 @@ Notice how we now instead output ``np.nan`` itself instead of a stringified form
 - Bug in :func:`to_string()` that broke column alignment when ``index=False`` and width of first column's values is greater than the width of first column's header (:issue:`16839`, :issue:`13032`)
 - Bug in :func:`DataFrame.to_csv` where a single level MultiIndex incorrectly wrote a tuple. Now just the value of the index is written (:issue:`19589`).
 - Bug in :meth:`HDFStore.append` when appending a :class:`DataFrame` with an empty string column and ``min_itemsize`` < 8 (:issue:`12242`)
+- Bug in :meth:`read_csv()` in which :class:`MultiIndex` index names were being improperly handled in the cases when they were not provided (:issue:`23484`)
 
 Plotting
 ^^^^^^^^
@@ -1305,13 +1309,15 @@ Groupby/Resample/Rolling
 - :func:`RollingGroupby.agg` and :func:`ExpandingGroupby.agg` now support multiple aggregation functions as parameters (:issue:`15072`)
 - Bug in :meth:`DataFrame.resample` and :meth:`Series.resample` when resampling by a weekly offset (``'W'``) across a DST transition (:issue:`9119`, :issue:`21459`)
 - Bug in :meth:`DataFrame.expanding` in which the ``axis`` argument was not being respected during aggregations (:issue:`23372`)
+- Bug in :meth:`pandas.core.groupby.DataFrameGroupBy.transform` which caused missing values when the input function can accept a :class:`DataFrame` but renames it (:issue:`23455`).
 
 Reshaping
 ^^^^^^^^^
 
 - Bug in :func:`pandas.concat` when joining resampled DataFrames with timezone aware index (:issue:`13783`)
 - Bug in :meth:`Series.combine_first` with ``datetime64[ns, tz]`` dtype which would return tz-naive result (:issue:`21469`)
 - Bug in :meth:`Series.where` and :meth:`DataFrame.where` with ``datetime64[ns, tz]`` dtype (:issue:`21546`)
+- Bug in :meth:`DataFrame.where` with an empty DataFrame and empty ``cond`` having non-bool dtype (:issue:`21947`)
 - Bug in :meth:`Series.mask` and :meth:`DataFrame.mask` with ``list`` conditionals (:issue:`21891`)
 - Bug in :meth:`DataFrame.replace` raises RecursionError when converting OutOfBounds ``datetime64[ns, tz]`` (:issue:`20380`)
 - :func:`pandas.core.groupby.GroupBy.rank` now raises a ``ValueError`` when an invalid value is passed for argument ``na_option`` (:issue:`22124`)

pandas/_libs/algos.pyx

@@ -128,11 +128,11 @@ def is_lexsorted(list_of_arrays: list) -> bint:
     nlevels = len(list_of_arrays)
     n = len(list_of_arrays[0])
 
-    cdef int64_t **vecs = <int64_t**> malloc(nlevels * sizeof(int64_t*))
+    cdef int64_t **vecs = <int64_t**>malloc(nlevels * sizeof(int64_t*))
     for i in range(nlevels):
         arr = list_of_arrays[i]
         assert arr.dtype.name == 'int64'
-        vecs[i] = <int64_t*> cnp.PyArray_DATA(arr)
+        vecs[i] = <int64_t*>cnp.PyArray_DATA(arr)
 
     # Assume uniqueness??
     with nogil:
@@ -409,7 +409,7 @@ def pad(ndarray[algos_t] old, ndarray[algos_t] new, limit=None):
     nleft = len(old)
     nright = len(new)
     indexer = np.empty(nright, dtype=np.int64)
-    indexer.fill(-1)
+    indexer[:] = -1
 
     if limit is None:
         lim = nright
@@ -607,7 +607,7 @@ def backfill(ndarray[algos_t] old, ndarray[algos_t] new, limit=None):
     nleft = len(old)
     nright = len(new)
     indexer = np.empty(nright, dtype=np.int64)
-    indexer.fill(-1)
+    indexer[:] = -1
 
     if limit is None:
         lim = nright

pandas/_libs/algos_rank_helper.pxi.in

@@ -4,9 +4,9 @@ Template for each `dtype` helper function for rank
 WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 """
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # rank_1d, rank_2d
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 
 {{py:
 

pandas/_libs/groupby.pyx

@@ -44,7 +44,7 @@ cdef inline float64_t median_linear(float64_t* a, int n) nogil:
         if na_count == n:
             return NaN
 
-        tmp = <float64_t*> malloc((n - na_count) * sizeof(float64_t))
+        tmp = <float64_t*>malloc((n - na_count) * sizeof(float64_t))
 
         j = 0
         for i in range(n):
@@ -121,7 +121,7 @@ def group_median_float64(ndarray[float64_t, ndim=2] out,
     counts[:] = _counts[1:]
 
     data = np.empty((K, N), dtype=np.float64)
-    ptr = <float64_t*> cnp.PyArray_DATA(data)
+    ptr = <float64_t*>cnp.PyArray_DATA(data)
 
     take_2d_axis1_float64_float64(values.T, indexer, out=data)
 
@@ -370,7 +370,7 @@ def group_any_all(ndarray[uint8_t] out,
     else:
         raise ValueError("'bool_func' must be either 'any' or 'all'!")
 
-    out.fill(1 - flag_val)
+    out[:] = 1 - flag_val
 
     with nogil:
         for i in range(N):

pandas/_libs/groupby_helper.pxi.in

@@ -8,9 +8,9 @@ cdef extern from "numpy/npy_math.h":
     double NAN "NPY_NAN"
 _int64_max = np.iinfo(np.int64).max
 
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 # group_add, group_prod, group_var, group_mean, group_ohlc
-#----------------------------------------------------------------------
+# ----------------------------------------------------------------------
 
 {{py:
 
@@ -246,7 +246,7 @@ def group_ohlc_{{name}}(ndarray[{{c_type}}, ndim=2] out,
     if K > 1:
         raise NotImplementedError("Argument 'values' must have only "
                                   "one dimension")
-    out.fill(np.nan)
+    out[:] = np.nan
 
     with nogil:
         for i in range(N):
@@ -629,10 +629,10 @@ def group_max(ndarray[groupby_t, ndim=2] out,
     maxx = np.empty_like(out)
     if groupby_t is int64_t:
         # Note: evaluated at compile-time
-        maxx.fill(-_int64_max)
+        maxx[:] = -_int64_max
         nan_val = iNaT
     else:
-        maxx.fill(-np.inf)
+        maxx[:] = -np.inf
         nan_val = NAN
 
     N, K = (<object>values).shape
@@ -691,10 +691,10 @@ def group_min(ndarray[groupby_t, ndim=2] out,
 
     minx = np.empty_like(out)
     if groupby_t is int64_t:
-        minx.fill(_int64_max)
+        minx[:] = _int64_max
         nan_val = iNaT
     else:
-        minx.fill(np.inf)
+        minx[:] = np.inf
         nan_val = NAN
 
     N, K = (<object>values).shape
@@ -747,9 +747,9 @@ def group_cummin(ndarray[groupby_t, ndim=2] out,
     N, K = (<object>values).shape
     accum = np.empty_like(values)
     if groupby_t is int64_t:
-        accum.fill(_int64_max)
+        accum[:] = _int64_max
     else:
-        accum.fill(np.inf)
+        accum[:] = np.inf
 
     with nogil:
         for i in range(N):
@@ -795,9 +795,9 @@ def group_cummax(ndarray[groupby_t, ndim=2] out,
     N, K = (<object>values).shape
     accum = np.empty_like(values)
     if groupby_t is int64_t:
-        accum.fill(-_int64_max)
+        accum[:] = -_int64_max
     else:
-        accum.fill(-np.inf)
+        accum[:] = -np.inf
 
     with nogil:
         for i in range(N):

pandas/_libs/hashing.pyx

@@ -54,8 +54,8 @@ def hash_object_array(object[:] arr, object key, object encoding='utf8'):
     n = len(arr)
 
     # create an array of bytes
-    vecs = <char **> malloc(n * sizeof(char *))
-    lens = <uint64_t*> malloc(n * sizeof(uint64_t))
+    vecs = <char **>malloc(n * sizeof(char *))
+    lens = <uint64_t*>malloc(n * sizeof(uint64_t))
 
     for i in range(n):
         val = arr[i]

pandas/_libs/hashtable_class_helper.pxi.in

@@ -590,13 +590,13 @@ cdef class StringHashTable(HashTable):
         cdef:
             Py_ssize_t i, n = len(values)
             ndarray[int64_t] labels = np.empty(n, dtype=np.int64)
-            int64_t *resbuf = <int64_t*> labels.data
+            int64_t *resbuf = <int64_t*>labels.data
             khiter_t k
             kh_str_t *table = self.table
             const char *v
             const char **vecs
 
-        vecs = <const char **> malloc(n * sizeof(char *))
+        vecs = <const char **>malloc(n * sizeof(char *))
         for i in range(n):
             val = values[i]
             v = util.get_c_string(val)
@@ -639,7 +639,7 @@ cdef class StringHashTable(HashTable):
             const char *v
             const char **vecs
 
-        vecs = <const char **> malloc(n * sizeof(char *))
+        vecs = <const char **>malloc(n * sizeof(char *))
         uindexer = np.empty(n, dtype=np.int64)
         for i in range(n):
             val = values[i]
@@ -674,7 +674,7 @@ cdef class StringHashTable(HashTable):
             int64_t[:] locs = np.empty(n, dtype=np.int64)
 
         # these by-definition *must* be strings
-        vecs = <char **> malloc(n * sizeof(char *))
+        vecs = <char **>malloc(n * sizeof(char *))
         for i in range(n):
             val = values[i]
 
@@ -707,7 +707,7 @@ cdef class StringHashTable(HashTable):
             khiter_t k
 
         # these by-definition *must* be strings
-        vecs = <const char **> malloc(n * sizeof(char *))
+        vecs = <const char **>malloc(n * sizeof(char *))
         for i in range(n):
             val = values[i]
 

pandas/_libs/join.pyx

@@ -212,7 +212,7 @@ def _get_result_indexer(sorter, indexer):
     else:
         # length-0 case
         res = np.empty(len(indexer), dtype=np.int64)
-        res.fill(-1)
+        res[:] = -1
 
     return res
 

pandas/_libs/lib.pyx

@@ -347,7 +347,7 @@ def get_reverse_indexer(ndarray[int64_t] indexer, Py_ssize_t length):
         int64_t idx
 
     rev_indexer = np.empty(length, dtype=np.int64)
-    rev_indexer.fill(-1)
+    rev_indexer[:] = -1
     for i in range(n):
         idx = indexer[i]
         if idx != -1:
@@ -1670,7 +1670,7 @@ cdef class TimedeltaValidator(TemporalValidator):
 
 
 # TODO: Not used outside of tests; remove?
-def is_timedelta_array(values: ndarray) -> bint:
+def is_timedelta_array(values: ndarray) -> bool:
     cdef:
         TimedeltaValidator validator = TimedeltaValidator(len(values),
                                                           skipna=True)
@@ -1683,7 +1683,7 @@ cdef class Timedelta64Validator(TimedeltaValidator):
 
 
 # TODO: Not used outside of tests; remove?
-def is_timedelta64_array(values: ndarray) -> bint:
+def is_timedelta64_array(values: ndarray) -> bool:
     cdef:
         Timedelta64Validator validator = Timedelta64Validator(len(values),
                                                               skipna=True)

pandas/_libs/missing.pyx

@@ -278,14 +278,14 @@ def isnaobj2d_old(ndarray arr):
     return result.view(np.bool_)
 
 
-cpdef bint isposinf_scalar(object val):
+def isposinf_scalar(val: object) -> bool:
     if util.is_float_object(val) and val == INF:
         return True
     else:
         return False
 
 
-cpdef bint isneginf_scalar(object val):
+def isneginf_scalar(val: object) -> bool:
     if util.is_float_object(val) and val == NEGINF:
         return True
     else: