Merge branch 'master' of https://github.com/pandas-dev/pandas into dlike8

Author: jbrockmendel

doc/source/whatsnew/v0.24.0.txt

@@ -688,6 +688,7 @@ Other API Changes
 - :meth:`DataFrame.corr` and :meth:`Series.corr` now raise a ``ValueError`` along with a helpful error message instead of a ``KeyError`` when supplied with an invalid method (:issue:`22298`)
 - :meth:`shift` will now always return a copy, instead of the previous behaviour of returning self when shifting by 0 (:issue:`22397`)
 - Slicing a single row of a DataFrame with multiple ExtensionArrays of the same type now preserves the dtype, rather than coercing to object (:issue:`22784`)
+- :class:`DateOffset` attribute `_cacheable` and method `_should_cache` have been removed (:issue:`23118`)
 
 .. _whatsnew_0240.deprecations:
 
@@ -844,7 +845,7 @@ Numeric
 - Bug in :class:`DataFrame` multiplication between boolean dtype and integer returning ``object`` dtype instead of integer dtype (:issue:`22047`, :issue:`22163`)
 - Bug in :meth:`DataFrame.apply` where, when supplied with a string argument and additional positional or keyword arguments (e.g. ``df.apply('sum', min_count=1)``), a ``TypeError`` was wrongly raised (:issue:`22376`)
 - Bug in :meth:`DataFrame.astype` to extension dtype may raise ``AttributeError`` (:issue:`22578`)
-
+- Bug in :class:`DataFrame` with ``timedelta64[ns]`` dtype arithmetic operations with ``ndarray`` with integer dtype incorrectly treating the narray as ``timedelta64[ns]`` dtype (:issue:`23114`)
 
 Strings
 ^^^^^^^

pandas/_libs/tslibs/offsets.pyx

@@ -282,11 +282,6 @@ class ApplyTypeError(TypeError):
     pass
 
 
-# TODO: unused.  remove?
-class CacheableOffset(object):
-    _cacheable = True
-
-
 # ---------------------------------------------------------------------
 # Base Classes
 
@@ -296,8 +291,6 @@ class _BaseOffset(object):
     and will (after pickle errors are resolved) go into a cdef class.
     """
     _typ = "dateoffset"
-    _normalize_cache = True
-    _cacheable = False
     _day_opt = None
     _attributes = frozenset(['n', 'normalize'])
 
@@ -386,10 +379,6 @@ class _BaseOffset(object):
         # that allows us to use methods that can go in a `cdef class`
         return self * 1
 
-    # TODO: this is never true.  fix it or get rid of it
-    def _should_cache(self):
-        return self.isAnchored() and self._cacheable
-
     def __repr__(self):
         className = getattr(self, '_outputName', type(self).__name__)
 

pandas/core/arrays/datetimes.py

@@ -13,7 +13,7 @@
     resolution as libresolution)
 
 from pandas.util._decorators import cache_readonly
-from pandas.errors import PerformanceWarning, AbstractMethodError
+from pandas.errors import PerformanceWarning
 from pandas import compat
 
 from pandas.core.dtypes.common import (
@@ -268,27 +268,22 @@ def _generate_range(cls, start, end, periods, freq, tz=None,
                 end, end.tz, start.tz, freq, tz
             )
         if freq is not None:
-            if cls._use_cached_range(freq, _normalized, start, end):
-                # Currently always False; never hit
-                # Should be reimplemented as a part of GH#17914
-                index = cls._cached_range(start, end, periods=periods,
-                                          freq=freq)
-            else:
-                index = _generate_regular_range(cls, start, end, periods, freq)
-
-                if tz is not None and getattr(index, 'tz', None) is None:
-                    arr = conversion.tz_localize_to_utc(
-                        ensure_int64(index.values),
-                        tz, ambiguous=ambiguous)
-
-                    index = cls(arr)
-
-                    # index is localized datetime64 array -> have to convert
-                    # start/end as well to compare
-                    if start is not None:
-                        start = start.tz_localize(tz).asm8
-                    if end is not None:
-                        end = end.tz_localize(tz).asm8
+            # TODO: consider re-implementing _cached_range; GH#17914
+            index = _generate_regular_range(cls, start, end, periods, freq)
+
+            if tz is not None and getattr(index, 'tz', None) is None:
+                arr = conversion.tz_localize_to_utc(
+                    ensure_int64(index.values),
+                    tz, ambiguous=ambiguous)
+
+                index = cls(arr)
+
+                # index is localized datetime64 array -> have to convert
+                # start/end as well to compare
+                if start is not None:
+                    start = start.tz_localize(tz).asm8
+                if end is not None:
+                    end = end.tz_localize(tz).asm8
         else:
             # Create a linearly spaced date_range in local time
             arr = np.linspace(start.value, end.value, periods)
@@ -303,16 +298,6 @@ def _generate_range(cls, start, end, periods, freq, tz=None,
 
         return cls._simple_new(index.values, freq=freq, tz=tz)
 
-    @classmethod
-    def _use_cached_range(cls, freq, _normalized, start, end):
-        # DatetimeArray is mutable, so is not cached
-        return False
-
-    @classmethod
-    def _cached_range(cls, start=None, end=None,
-                      periods=None, freq=None, **kwargs):
-        raise AbstractMethodError(cls)
-
     # -----------------------------------------------------------------
     # Descriptive Properties
 

pandas/core/generic.py

@@ -1795,6 +1795,10 @@ def __round__(self, decimals=0):
     # ----------------------------------------------------------------------
     # Array Interface
 
+    # This is also set in IndexOpsMixin
+    # GH#23114 Ensure ndarray.__op__(DataFrame) returns NotImplemented
+    __array_priority__ = 1000
+
     def __array__(self, dtype=None):
         return com.values_from_object(self)
 

pandas/core/indexes/datetimes.py

@@ -40,7 +40,7 @@
     DatelikeOps, TimelikeOps, DatetimeIndexOpsMixin,
     wrap_field_accessor, wrap_array_method)
 from pandas.tseries.offsets import (
-    generate_range, CDay, prefix_mapping)
+    CDay, prefix_mapping)
 
 from pandas.core.tools.timedeltas import to_timedelta
 from pandas.util._decorators import Appender, cache_readonly, Substitution
@@ -317,13 +317,6 @@ def __new__(cls, data=None,
 
         return subarr._deepcopy_if_needed(ref_to_data, copy)
 
-    @classmethod
-    def _use_cached_range(cls, freq, _normalized, start, end):
-        # Note: This always returns False
-        return (freq._should_cache() and
-                not (freq._normalize_cache and not _normalized) and
-                _naive_in_cache_range(start, end))
-
     def _convert_for_op(self, value):
         """ Convert value to be insertable to ndarray """
         if self._has_same_tz(value):
@@ -380,71 +373,6 @@ def tz(self, value):
         raise AttributeError("Cannot directly set timezone. Use tz_localize() "
                              "or tz_convert() as appropriate")
 
-    @classmethod
-    def _cached_range(cls, start=None, end=None, periods=None, freq=None,
-                      name=None):
-        if start is None and end is None:
-            # I somewhat believe this should never be raised externally
-            raise TypeError('Must specify either start or end.')
-        if start is not None:
-            start = Timestamp(start)
-        if end is not None:
-            end = Timestamp(end)
-        if (start is None or end is None) and periods is None:
-            raise TypeError(
-                'Must either specify period or provide both start and end.')
-
-        if freq is None:
-            # This can't happen with external-facing code
-            raise TypeError('Must provide freq.')
-
-        drc = _daterange_cache
-        if freq not in _daterange_cache:
-            xdr = generate_range(offset=freq, start=_CACHE_START,
-                                 end=_CACHE_END)
-
-            arr = tools.to_datetime(list(xdr), box=False)
-
-            cachedRange = DatetimeIndex._simple_new(arr)
-            cachedRange.freq = freq
-            cachedRange = cachedRange.tz_localize(None)
-            cachedRange.name = None
-            drc[freq] = cachedRange
-        else:
-            cachedRange = drc[freq]
-
-        if start is None:
-            if not isinstance(end, Timestamp):
-                raise AssertionError('end must be an instance of Timestamp')
-
-            end = freq.rollback(end)
-
-            endLoc = cachedRange.get_loc(end) + 1
-            startLoc = endLoc - periods
-        elif end is None:
-            if not isinstance(start, Timestamp):
-                raise AssertionError('start must be an instance of Timestamp')
-
-            start = freq.rollforward(start)
-
-            startLoc = cachedRange.get_loc(start)
-            endLoc = startLoc + periods
-        else:
-            if not freq.onOffset(start):
-                start = freq.rollforward(start)
-
-            if not freq.onOffset(end):
-                end = freq.rollback(end)
-
-            startLoc = cachedRange.get_loc(start)
-            endLoc = cachedRange.get_loc(end) + 1
-
-        indexSlice = cachedRange[startLoc:endLoc]
-        indexSlice.name = name
-        indexSlice.freq = freq
-
-        return indexSlice
-
     def _mpl_repr(self):
         # how to represent ourselves to matplotlib
         return libts.ints_to_pydatetime(self.asi8, self.tz)
@@ -802,22 +730,19 @@ def _fast_union(self, other):
         else:
             left, right = other, self
 
-        left_start, left_end = left[0], left[-1]
+        left_end = left[-1]
         right_end = right[-1]
 
-        if not self.freq._should_cache():
-            # concatenate dates
-            if left_end < right_end:
-                loc = right.searchsorted(left_end, side='right')
-                right_chunk = right.values[loc:]
-                dates = _concat._concat_compat((left.values, right_chunk))
-                return self._shallow_copy(dates)
-            else:
-                return left
+        # TODO: consider re-implementing freq._should_cache for fastpath
+
+        # concatenate dates
+        if left_end < right_end:
+            loc = right.searchsorted(left_end, side='right')
+            right_chunk = right.values[loc:]
+            dates = _concat._concat_compat((left.values, right_chunk))
+            return self._shallow_copy(dates)
         else:
-            return type(self)(start=left_start,
-                              end=max(left_end, right_end),
-                              freq=left.freq)
+            return left
 
     def _wrap_union_result(self, other, result):
         name = self.name if self.name == other.name else None
@@ -1694,21 +1619,6 @@ def cdate_range(start=None, end=None, periods=None, freq='C', tz=None,
                          closed=closed, **kwargs)
 
 
-_CACHE_START = Timestamp(datetime(1950, 1, 1))
-_CACHE_END = Timestamp(datetime(2030, 1, 1))
-
-_daterange_cache = {}
-
-
-def _naive_in_cache_range(start, end):
-    if start is None or end is None:
-        return False
-    else:
-        if start.tzinfo is not None or end.tzinfo is not None:
-            return False
-        return start > _CACHE_START and end < _CACHE_END
-
-
 def _time_to_micros(time):
     seconds = time.hour * 60 * 60 + 60 * time.minute + time.second
     return 1000000 * seconds + time.microsecond

pandas/io/formats/style.py

@@ -64,6 +64,11 @@ class Styler(object):
         a unique identifier to avoid CSS collisions; generated automatically
     caption: str, default None
         caption to attach to the table
+    cell_ids: bool, default True
+        If True, each cell will have an ``id`` attribute in their HTML tag.
+        The ``id`` takes the form ``T_<uuid>_row<num_row>_col<num_col>``
+        where ``<uuid>`` is the unique identifier, ``<num_row>`` is the row
+        number and ``<num_col>`` is the column number.
 
     Attributes
     ----------
@@ -112,7 +117,7 @@ class Styler(object):
     template = env.get_template("html.tpl")
 
     def __init__(self, data, precision=None, table_styles=None, uuid=None,
-                 caption=None, table_attributes=None):
+                 caption=None, table_attributes=None, cell_ids=True):
         self.ctx = defaultdict(list)
         self._todo = []
 
@@ -136,6 +141,7 @@ def __init__(self, data, precision=None, table_styles=None, uuid=None,
         self.table_attributes = table_attributes
         self.hidden_index = False
         self.hidden_columns = []
+        self.cell_ids = cell_ids
 
         # display_funcs maps (row, col) -> formatting function
 
@@ -306,14 +312,16 @@ def format_attr(pair):
                 cs.extend(cell_context.get("data", {}).get(r, {}).get(c, []))
                 formatter = self._display_funcs[(r, c)]
                 value = self.data.iloc[r, c]
-                row_es.append({
-                    "type": "td",
-                    "value": value,
-                    "class": " ".join(cs),
-                    "id": "_".join(cs[1:]),
-                    "display_value": formatter(value),
-                    "is_visible": (c not in hidden_columns)
-                })
+                row_dict = {"type": "td",
+                            "value": value,
+                            "class": " ".join(cs),
+                            "display_value": formatter(value),
+                            "is_visible": (c not in hidden_columns)}
+                # only add an id if the cell has a style
+                if (self.cell_ids or
+                        not(len(ctx[r, c]) == 1 and ctx[r, c][0] == '')):
+                    row_dict["id"] = "_".join(cs[1:])
+                row_es.append(row_dict)
                 props = []
                 for x in ctx[r, c]:
                     # have to handle empty styles like ['']

pandas/io/formats/templates/html.tpl

@@ -50,17 +50,17 @@
 {%- endblock thead %} 
 {%- block tbody %} 
 <tbody> 
-    {%- block before_rows %}{%- endblock before_rows %}
-    {%- for r in body %} 
-    {%- block tr scoped %}
-    <tr> 
-        {%- for c in r %} 
-        {%- if c.is_visible != False %} 
-        <{{ c.type }} id="T_{{ uuid }}{{ c.id }}" class="{{ c.class }}" {{ c.attributes|join(" ") }}>{{ c.display_value }}</{{ c.type }}>
-        {%- endif %} 
-        {%- endfor %} 
-    </tr> 
-    {%- endblock tr %}
+    {% block before_rows %}{% endblock before_rows %}
+    {% for r in body %}
+    {% block tr scoped %}
+    <tr>
+        {% for c in r %}
+        {% if c.is_visible != False %}
+        <{{ c.type }} {% if c.id is defined -%} id="T_{{ uuid }}{{ c.id }}" {%- endif %} class="{{ c.class }}" {{ c.attributes|join(" ") }}>{{ c.display_value }}</{{ c.type }}>
+        {% endif %}
+        {%- endfor %}
+    </tr>
+    {% endblock tr %}
     {%- endfor %} 
     {%- block after_rows %}{%- endblock after_rows %}
 </tbody> 

pandas/tests/arithmetic/test_numeric.py

@@ -156,7 +156,7 @@ def test_numeric_arr_rdiv_tdscalar(self, three_days, numeric_idx, box):
         if box is not pd.Index and broken:
             # np.timedelta64(3, 'D') / 2 == np.timedelta64(1, 'D')
             raise pytest.xfail("timedelta64 not converted to nanos; "
-                               "Tick division not imlpemented")
+                               "Tick division not implemented")
 
         expected = TimedeltaIndex(['3 Days', '36 Hours'])
 

pandas/tests/arithmetic/test_object.py

@@ -140,9 +140,6 @@ def test_objarr_radd_str_invalid(self, dtype, data, box):
                                     operator.sub, ops.rsub])
     def test_objarr_add_invalid(self, op, box):
         # invalid ops
-        if box is pd.DataFrame and op is ops.radd:
-            pytest.xfail(reason="DataFrame op incorrectly casts the np.array"
-                                "case to M8[ns]")
 
         obj_ser = tm.makeObjectSeries()
         obj_ser.name = 'objects'