Merge branch 'master' of https://github.com/pandas-dev/pandas into tslibs-offsets6

Author: jbrockmendel

doc/source/whatsnew/v0.21.1.txt

@@ -73,7 +73,7 @@ Indexing
 - Bug in a boolean comparison of a ``datetime.datetime`` and a ``datetime64[ns]`` dtype Series (:issue:`17965`)
 - Bug where a ``MultiIndex`` with more than a million records was not raising ``AttributeError`` when trying to access a missing attribute (:issue:`18165`)
 - Bug in :class:`IntervalIndex` constructor when a list of intervals is passed with non-default ``closed`` (:issue:`18334`)
--
+- Bug in ``Index.putmask`` when an invalid mask passed (:issue:`18368`)
 -
 
 I/O
@@ -103,7 +103,7 @@ Groupby/Resample/Rolling
 - Bug in ``DataFrame.resample(...).apply(...)`` when there is a callable that returns different columns (:issue:`15169`)
 - Bug in ``DataFrame.resample(...)`` when there is a time change (DST) and resampling frequecy is 12h or higher (:issue:`15549`)
 - Bug in ``pd.DataFrameGroupBy.count()`` when counting over a datetimelike column (:issue:`13393`)
--
+- Bug in ``rolling.var`` where calculation is inaccurate with a zero-valued array (:issue:`18430`)
 -
 -
 

doc/source/whatsnew/v0.22.0.txt

@@ -79,7 +79,7 @@ Other API Changes
 - :func:`Dataframe.unstack` will now default to filling with ``np.nan`` for ``object`` columns. (:issue:`12815`)
 - :class:`IntervalIndex` constructor will raise if the ``closed`` parameter conflicts with how the input data is inferred to be closed (:issue:`18421`)
 - Inserting missing values into indexes will work for all types of indexes and automatically insert the correct type of missing value (``NaN``, ``NaT``, etc.) regardless of the type passed in (:issue:`18295`)
-
+- Restricted ``DateOffset`` keyword arguments. Previously, ``DateOffset`` subclasses allowed arbitrary keyword arguments which could lead to unexpected behavior. Now, only valid arguments will be accepted. (:issue:`17176`, :issue:`18226`).
 
 .. _whatsnew_0220.deprecations:
 
@@ -144,6 +144,7 @@ Indexing
 - Bug in :func:`Series.truncate` which raises ``TypeError`` with a monotonic ``PeriodIndex`` (:issue:`17717`)
 - Bug in :func:`DataFrame.groupby` where tuples were interpreted as lists of keys rather than as keys (:issue:`17979`, :issue:`18249`)
 - Bug in :func:`MultiIndex.remove_unused_levels`` which would fill nan values (:issue:`18417`)
+- Bug in :func:`MultiIndex.from_tuples`` which would fail to take zipped tuples in python3 (:issue:`18434`)
 - Bug in :class:`IntervalIndex` where empty and purely NA data was constructed inconsistently depending on the construction method (:issue:`18421`)
 -
 

pandas/_libs/tslibs/offsets.pyx

@@ -259,7 +259,7 @@ def _validate_business_time(t_input):
 # ---------------------------------------------------------------------
 # Constructor Helpers
 
-_rd_kwds = set([
+relativedelta_kwds = set([
     'years', 'months', 'weeks', 'days',
     'year', 'month', 'week', 'day', 'weekday',
     'hour', 'minute', 'second', 'microsecond',
@@ -404,6 +404,33 @@ class _BaseOffset(object):
         # will raise NotImplementedError.
         return get_day_of_month(other, self._day_opt)
 
+    def _validate_n(self, n):
+        """
+        Require that `n` be a nonzero integer.
+
+        Parameters
+        ----------
+        n : int
+
+        Returns
+        -------
+        nint : int
+
+        Raises
+        ------
+        TypeError if `int(n)` raises
+        ValueError if n != int(n)
+        """
+        try:
+            nint = int(n)
+        except (ValueError, TypeError):
+            raise TypeError('`n` argument must be an integer, '
+                            'got {ntype}'.format(ntype=type(n)))
+        if n != nint:
+            raise ValueError('`n` argument must be an integer, '
+                             'got {n}'.format(n=n))
+        return nint
+
 
 class BaseOffset(_BaseOffset):
     # Here we add __rfoo__ methods that don't play well with cdef classes

pandas/_libs/window.pyx

@@ -661,9 +661,11 @@ cdef inline void add_var(double val, double *nobs, double *mean_x,
     if val == val:
         nobs[0] = nobs[0] + 1
 
-        delta = (val - mean_x[0])
+        # a part of Welford's method for the online variance-calculation
+        # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
+        delta = val - mean_x[0]
         mean_x[0] = mean_x[0] + delta / nobs[0]
-        ssqdm_x[0] = ssqdm_x[0] + delta * (val - mean_x[0])
+        ssqdm_x[0] = ssqdm_x[0] + ((nobs[0] - 1) * delta ** 2) / nobs[0]
 
 
 cdef inline void remove_var(double val, double *nobs, double *mean_x,
@@ -675,9 +677,11 @@ cdef inline void remove_var(double val, double *nobs, double *mean_x,
     if val == val:
         nobs[0] = nobs[0] - 1
         if nobs[0]:
-            delta = (val - mean_x[0])
+            # a part of Welford's method for the online variance-calculation
+            # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
+            delta = val - mean_x[0]
             mean_x[0] = mean_x[0] - delta / nobs[0]
-            ssqdm_x[0] = ssqdm_x[0] - delta * (val - mean_x[0])
+            ssqdm_x[0] = ssqdm_x[0] - ((nobs[0] + 1) * delta ** 2) / nobs[0]
         else:
             mean_x[0] = 0
             ssqdm_x[0] = 0
@@ -689,7 +693,7 @@ def roll_var(ndarray[double_t] input, int64_t win, int64_t minp,
     Numerically stable implementation using Welford's method.
     """
     cdef:
-        double val, prev, mean_x = 0, ssqdm_x = 0, nobs = 0, delta
+        double val, prev, mean_x = 0, ssqdm_x = 0, nobs = 0, delta, mean_x_old
         int64_t s, e
         bint is_variable
         Py_ssize_t i, j, N
@@ -749,6 +753,9 @@ def roll_var(ndarray[double_t] input, int64_t win, int64_t minp,
                 add_var(input[i], &nobs, &mean_x, &ssqdm_x)
                 output[i] = calc_var(minp, ddof, nobs, ssqdm_x)
 
+            # a part of Welford's method for the online variance-calculation
+            # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
+
             # After the first window, observations can both be added and
             # removed
             for i from win <= i < N:
@@ -760,10 +767,12 @@ def roll_var(ndarray[double_t] input, int64_t win, int64_t minp,
 
                         # Adding one observation and removing another one
                         delta = val - prev
-                        prev -= mean_x
+                        mean_x_old = mean_x
+
                         mean_x += delta / nobs
-                        val -= mean_x
-                        ssqdm_x += (val + prev) * delta
+                        ssqdm_x += ((nobs - 1) * val
+                                    + (nobs + 1) * prev
+                                    - 2 * nobs * mean_x_old) * delta / nobs
 
                     else:
                         add_var(val, &nobs, &mean_x, &ssqdm_x)

pandas/core/indexes/base.py

@@ -1939,7 +1939,10 @@ def putmask(self, mask, value):
         try:
             np.putmask(values, mask, self._convert_for_op(value))
             return self._shallow_copy(values)
-        except (ValueError, TypeError):
+        except (ValueError, TypeError) as err:
+            if is_object_dtype(self):
+                raise err
+
             # coerces to object
             return self.astype(object).putmask(mask, value)
 

pandas/core/indexes/multi.py

@@ -1162,6 +1162,11 @@ def from_arrays(cls, arrays, sortorder=None, names=None):
         MultiIndex.from_product : Make a MultiIndex from cartesian product
                                   of iterables
         """
+        if not is_list_like(arrays):
+            raise TypeError("Input must be a list / sequence of array-likes.")
+        elif is_iterator(arrays):
+            arrays = list(arrays)
+
         # Check if lengths of all arrays are equal or not,
         # raise ValueError, if not
         for i in range(1, len(arrays)):
@@ -1206,6 +1211,11 @@ def from_tuples(cls, tuples, sortorder=None, names=None):
         MultiIndex.from_product : Make a MultiIndex from cartesian product
                                   of iterables
         """
+        if not is_list_like(tuples):
+            raise TypeError('Input must be a list / sequence of tuple-likes.')
+        elif is_iterator(tuples):
+            tuples = list(tuples)
+
         if len(tuples) == 0:
             if names is None:
                 msg = 'Cannot infer number of levels from empty list'
@@ -1260,6 +1270,11 @@ def from_product(cls, iterables, sortorder=None, names=None):
         from pandas.core.categorical import _factorize_from_iterables
         from pandas.core.reshape.util import cartesian_product
 
+        if not is_list_like(iterables):
+            raise TypeError("Input must be a list / sequence of iterables.")
+        elif is_iterator(iterables):
+            iterables = list(iterables)
+
         labels, levels = _factorize_from_iterables(iterables)
         labels = cartesian_product(labels)
         return MultiIndex(levels, labels, sortorder=sortorder, names=names)

pandas/tests/indexes/common.py

@@ -1032,3 +1032,16 @@ def test_map(self):
 
         dict_map = {}
         tm.assert_index_equal(index.map(dict_map), nan_index)
+
+    def test_putmask_with_wrong_mask(self):
+        # GH18368
+        index = self.create_index()
+
+        with pytest.raises(ValueError):
+            index.putmask(np.ones(len(index) + 1, np.bool), 1)
+
+        with pytest.raises(ValueError):
+            index.putmask(np.ones(len(index) - 1, np.bool), 1)
+
+        with pytest.raises(ValueError):
+            index.putmask('foo', 1)

pandas/tests/indexes/test_multi.py

@@ -672,15 +672,31 @@ def test_from_arrays(self):
         for lev, lab in zip(self.index.levels, self.index.labels):
             arrays.append(np.asarray(lev).take(lab))
 
-        result = MultiIndex.from_arrays(arrays)
-        assert list(result) == list(self.index)
+        # list of arrays as input
+        result = MultiIndex.from_arrays(arrays, names=self.index.names)
+        tm.assert_index_equal(result, self.index)
 
         # infer correctly
         result = MultiIndex.from_arrays([[pd.NaT, Timestamp('20130101')],
                                          ['a', 'b']])
         assert result.levels[0].equals(Index([Timestamp('20130101')]))
         assert result.levels[1].equals(Index(['a', 'b']))
 
+    def test_from_arrays_iterator(self):
+        # GH 18434
+        arrays = []
+        for lev, lab in zip(self.index.levels, self.index.labels):
+            arrays.append(np.asarray(lev).take(lab))
+
+        # iterator as input
+        result = MultiIndex.from_arrays(iter(arrays), names=self.index.names)
+        tm.assert_index_equal(result, self.index)
+
+        # invalid iterator input
+        with tm.assert_raises_regex(
+                TypeError, "Input must be a list / sequence of array-likes."):
+            MultiIndex.from_arrays(0)
+
     def test_from_arrays_index_series_datetimetz(self):
         idx1 = pd.date_range('2015-01-01 10:00', freq='D', periods=3,
                              tz='US/Eastern')
@@ -825,7 +841,25 @@ def test_from_product(self):
         expected = MultiIndex.from_tuples(tuples, names=names)
 
         tm.assert_index_equal(result, expected)
-        assert result.names == names
+
+    def test_from_product_iterator(self):
+        # GH 18434
+        first = ['foo', 'bar', 'buz']
+        second = ['a', 'b', 'c']
+        names = ['first', 'second']
+        tuples = [('foo', 'a'), ('foo', 'b'), ('foo', 'c'), ('bar', 'a'),
+                  ('bar', 'b'), ('bar', 'c'), ('buz', 'a'), ('buz', 'b'),
+                  ('buz', 'c')]
+        expected = MultiIndex.from_tuples(tuples, names=names)
+
+        # iterator as input
+        result = MultiIndex.from_product(iter([first, second]), names=names)
+        tm.assert_index_equal(result, expected)
+
+        # Invalid non-iterable input
+        with tm.assert_raises_regex(
+                TypeError, "Input must be a list / sequence of iterables."):
+            MultiIndex.from_product(0)
 
     def test_from_product_empty(self):
         # 0 levels
@@ -1725,8 +1759,28 @@ def test_from_tuples(self):
                                'from empty list',
                                MultiIndex.from_tuples, [])
 
-        idx = MultiIndex.from_tuples(((1, 2), (3, 4)), names=['a', 'b'])
-        assert len(idx) == 2
+        expected = MultiIndex(levels=[[1, 3], [2, 4]],
+                              labels=[[0, 1], [0, 1]],
+                              names=['a', 'b'])
+
+        # input tuples
+        result = MultiIndex.from_tuples(((1, 2), (3, 4)), names=['a', 'b'])
+        tm.assert_index_equal(result, expected)
+
+    def test_from_tuples_iterator(self):
+        # GH 18434
+        # input iterator for tuples
+        expected = MultiIndex(levels=[[1, 3], [2, 4]],
+                              labels=[[0, 1], [0, 1]],
+                              names=['a', 'b'])
+
+        result = MultiIndex.from_tuples(zip([1, 3], [2, 4]), names=['a', 'b'])
+        tm.assert_index_equal(result, expected)
+
+        # input non-iterables
+        with tm.assert_raises_regex(
+                TypeError, 'Input must be a list / sequence of tuple-likes.'):
+            MultiIndex.from_tuples(0)
 
     def test_from_tuples_empty(self):
         # GH 16777

pandas/tests/test_window.py

@@ -2482,6 +2482,14 @@ def test_rolling_corr_pairwise(self):
         self._check_pairwise_moment('rolling', 'corr', window=10,
                                     min_periods=5)
 
+    @pytest.mark.parametrize('window', range(7))
+    def test_rolling_corr_with_zero_variance(self, window):
+        # GH 18430
+        s = pd.Series(np.zeros(20))
+        other = pd.Series(np.arange(20))
+
+        assert s.rolling(window=window).corr(other=other).isna().all()
+
     def _check_pairwise_moment(self, dispatch, name, **kwargs):
         def get_result(obj, obj2=None):
             return getattr(getattr(obj, dispatch)(**kwargs), name)(obj2)

pandas/tests/tseries/offsets/conftest.py

@@ -7,6 +7,19 @@ def offset_types(request):
     return request.param
 
 
+@pytest.fixture(params=[getattr(offsets, o) for o in offsets.__all__ if
+                        issubclass(getattr(offsets, o), offsets.MonthOffset)
+                        and o != 'MonthOffset'])
+def month_classes(request):
+    return request.param
+
+
+@pytest.fixture(params=[getattr(offsets, o) for o in offsets.__all__ if
+                        issubclass(getattr(offsets, o), offsets.Tick)])
+def tick_classes(request):
+    return request.param
+
+
 @pytest.fixture(params=[None, 'UTC', 'Asia/Tokyo', 'US/Eastern',
                         'dateutil/Asia/Tokyo', 'dateutil/US/Pacific'])
 def tz(request):

pandas/tests/tseries/offsets/test_offsets.py

@@ -17,6 +17,7 @@
                                         get_offset, get_standard_freq)
 from pandas.core.indexes.datetimes import (
     _to_m8, DatetimeIndex, _daterange_cache)
+import pandas._libs.tslibs.offsets as liboffsets
 from pandas._libs.tslibs.offsets import WeekDay, CacheableOffset
 from pandas.tseries.offsets import (BDay, CDay, BQuarterEnd, BMonthEnd,
                                     BusinessHour, WeekOfMonth, CBMonthEnd,
@@ -4682,9 +4683,45 @@ def test_all_offset_classes(self, tup):
         assert first == second
 
 
+# ---------------------------------------------------------------------
 def test_get_offset_day_error():
     # subclass of _BaseOffset must override _day_opt attribute, or we should
     # get a NotImplementedError
 
     with pytest.raises(NotImplementedError):
         DateOffset()._get_offset_day(datetime.now())
+
+
+@pytest.mark.parametrize('kwd', sorted(list(liboffsets.relativedelta_kwds)))
+def test_valid_month_attributes(kwd, month_classes):
+    # GH#18226
+    cls = month_classes
+    # check that we cannot create e.g. MonthEnd(weeks=3)
+    with pytest.raises(TypeError):
+        cls(**{kwd: 3})
+
+
+@pytest.mark.parametrize('kwd', sorted(list(liboffsets.relativedelta_kwds)))
+def test_valid_tick_attributes(kwd, tick_classes):
+    # GH#18226
+    cls = tick_classes
+    # check that we cannot create e.g. Hour(weeks=3)
+    with pytest.raises(TypeError):
+        cls(**{kwd: 3})
+
+
+def test_validate_n_error():
+    with pytest.raises(TypeError):
+        DateOffset(n='Doh!')
+
+    with pytest.raises(TypeError):
+        MonthBegin(n=timedelta(1))
+
+    with pytest.raises(TypeError):
+        BDay(n=np.array([1, 2], dtype=np.int64))
+
+
+def test_require_integers(offset_types):
+    cls = offset_types
+    with pytest.raises(ValueError):
+        cls(n=1.5)