Merge branch 'GH8302' of https://github.com/staple/pandas into staple-GH8302

Conflicts:
	doc/source/whatsnew/v0.15.2.txt

Author: jreback

doc/source/whatsnew/v0.15.2.txt

@@ -22,6 +22,20 @@ API changes
 
 - Bug in concat of Series with ``category`` dtype which were coercing to ``object``. (:issue:`8641`)
 
+- ``Series.all`` and ``Series.any`` now support the ``level`` and ``skipna`` parameters. ``Series.all``, ``Series.any``, ``Index.all``, and ``Index.any`` no longer support the ``out`` and ``keepdims`` parameters, which existed for compatibility with ndarray. Various index types no longer support the ``all`` and ``any`` aggregation functions and will now raise ``TypeError``. (:issue:`8302`):
+
+  .. ipython:: python
+
+     s = pd.Series([False, True, False], index=[0, 0, 1])
+     s.any(level=0)
+
+- ``Panel`` now supports the ``all`` and ``any`` aggregation functions. (:issue:`8302`):
+
+  .. ipython:: python
+
+     p = pd.Panel(np.random.rand(2, 5, 4) > 0.1)
+     p.all()
+
 .. _whatsnew_0152.enhancements:
 
 Enhancements
@@ -46,3 +60,4 @@ Bug Fixes
 - Bug in ``groupby`` signatures that didn't include *args or **kwargs (:issue:`8733`).
 - ``io.data.Options`` now raises ``RemoteDataError`` when no expiry dates are available from Yahoo and when it receives no data from Yahoo (:issue:`8761`), (:issue:`8783`).
 - Bug in slicing a multi-index with an empty list and at least one boolean indexer (:issue:`8781`)
+- ``io.data.Options`` now raises ``RemoteDataError`` when no expiry dates are available from Yahoo (:issue:`8761`).

pandas/core/base.py

@@ -268,18 +268,6 @@ def __unicode__(self):
                                  quote_strings=True)
         return "%s(%s, dtype='%s')" % (type(self).__name__, prepr, self.dtype)
 
-def _unbox(func):
-    @Appender(func.__doc__)
-    def f(self, *args, **kwargs):
-        result = func(self.values, *args, **kwargs)
-        from pandas.core.index import Index
-        if isinstance(result, (np.ndarray, com.ABCSeries, Index)) and result.ndim == 0:
-            # return NumPy type
-            return result.dtype.type(result.item())
-        else:  # pragma: no cover
-            return result
-    f.__name__ = func.__name__
-    return f
 
 class IndexOpsMixin(object):
     """ common ops mixin to support a unified inteface / docs for Series / Index """
@@ -528,12 +516,6 @@ def duplicated(self, take_last=False):
             from pandas.core.index import Index
             return Index(duplicated)
 
-    #----------------------------------------------------------------------
-    # unbox reductions
-
-    all = _unbox(np.ndarray.all)
-    any = _unbox(np.ndarray.any)
-
     #----------------------------------------------------------------------
     # abstracts
 

pandas/core/frame.py

@@ -4133,68 +4133,6 @@ def _count_level(self, level, axis=0, numeric_only=False):
         else:
             return result
 
-    def any(self, axis=None, bool_only=None, skipna=True, level=None,
-            **kwargs):
-        """
-        Return whether any element is True over requested axis.
-        %(na_action)s
-
-        Parameters
-        ----------
-        axis : {0, 1}
-            0 for row-wise, 1 for column-wise
-        skipna : boolean, default True
-            Exclude NA/null values. If an entire row/column is NA, the result
-            will be NA
-        level : int or level name, default None
-            If the axis is a MultiIndex (hierarchical), count along a
-            particular level, collapsing into a DataFrame
-        bool_only : boolean, default None
-            Only include boolean data.
-
-        Returns
-        -------
-        any : Series (or DataFrame if level specified)
-        """
-        if axis is None:
-            axis = self._stat_axis_number
-        if level is not None:
-            return self._agg_by_level('any', axis=axis, level=level,
-                                      skipna=skipna)
-        return self._reduce(nanops.nanany, 'any', axis=axis, skipna=skipna,
-                            numeric_only=bool_only, filter_type='bool')
-
-    def all(self, axis=None, bool_only=None, skipna=True, level=None,
-            **kwargs):
-        """
-        Return whether all elements are True over requested axis.
-        %(na_action)s
-
-        Parameters
-        ----------
-        axis : {0, 1}
-            0 for row-wise, 1 for column-wise
-        skipna : boolean, default True
-            Exclude NA/null values. If an entire row/column is NA, the result
-            will be NA
-        level : int or level name, default None
-            If the axis is a MultiIndex (hierarchical), count along a
-            particular level, collapsing into a DataFrame
-        bool_only : boolean, default None
-            Only include boolean data.
-
-        Returns
-        -------
-        any : Series (or DataFrame if level specified)
-        """
-        if axis is None:
-            axis = self._stat_axis_number
-        if level is not None:
-            return self._agg_by_level('all', axis=axis, level=level,
-                                      skipna=skipna)
-        return self._reduce(nanops.nanall, 'all', axis=axis, skipna=skipna,
-                            numeric_only=bool_only, filter_type='bool')
-
     def _reduce(self, op, name, axis=0, skipna=True, numeric_only=None,
                 filter_type=None, **kwds):
         axis = self._get_axis_number(axis)

pandas/core/generic.py

@@ -3888,6 +3888,7 @@ def _add_numeric_operations(cls):
         ])
         name = (cls._constructor_sliced.__name__
                 if cls._AXIS_LEN > 1 else 'scalar')
+
         _num_doc = """
 
 %(desc)s
@@ -3905,6 +3906,27 @@ def _add_numeric_operations(cls):
     Include only float, int, boolean data. If None, will attempt to use
     everything, then use only numeric data
 
+Returns
+-------
+%(outname)s : """ + name + " or " + cls.__name__ + " (if level specified)\n"
+
+        _bool_doc = """
+
+%(desc)s
+
+Parameters
+----------
+axis : """ + axis_descr + """
+skipna : boolean, default True
+    Exclude NA/null values. If an entire row/column is NA, the result
+    will be NA
+level : int or level name, default None
+        If the axis is a MultiIndex (hierarchical), count along a
+        particular level, collapsing into a """ + name + """
+bool_only : boolean, default None
+    Include only boolean data. If None, will attempt to use everything,
+    then use only boolean data
+
 Returns
 -------
 %(outname)s : """ + name + " or " + cls.__name__ + " (if level specified)\n"
@@ -3971,6 +3993,36 @@ def stat_func(self, axis=None, skipna=None, level=None,
 want the *index* of the minimum, use ``idxmin``. This is the
 equivalent of the ``numpy.ndarray`` method ``argmin``.""", nanops.nanmin)
 
+        def _make_logical_function(name, desc, f):
+
+            @Substitution(outname=name, desc=desc)
+            @Appender(_bool_doc)
+            def logical_func(self, axis=None, bool_only=None, skipna=None,
+                             level=None, **kwargs):
+                if skipna is None:
+                    skipna = True
+                if axis is None:
+                    axis = self._stat_axis_number
+                if level is not None:
+                    if bool_only is not None:
+                        raise NotImplementedError(
+                            "Option bool_only is not implemented with option "
+                            "level.")
+                    return self._agg_by_level(name, axis=axis, level=level,
+                                              skipna=skipna)
+                return self._reduce(f, axis=axis, skipna=skipna,
+                                    numeric_only=bool_only, filter_type='bool',
+                                    name=name)
+            logical_func.__name__ = name
+            return logical_func
+
+        cls.any = _make_logical_function(
+            'any', 'Return whether any element is True over requested axis',
+            nanops.nanany)
+        cls.all = _make_logical_function(
+            'all', 'Return whether all elements are True over requested axis',
+            nanops.nanall)
+
         @Substitution(outname='mad',
                       desc="Return the mean absolute deviation of the values "
                            "for the requested axis")

pandas/core/index.py

@@ -14,7 +14,8 @@
 import pandas.index as _index
 from pandas.lib import Timestamp, Timedelta, is_datetime_array
 from pandas.core.base import PandasObject, FrozenList, FrozenNDArray, IndexOpsMixin, _shared_docs
-from pandas.util.decorators import Appender, cache_readonly, deprecate
+from pandas.util.decorators import (Appender, Substitution, cache_readonly,
+                                    deprecate)
 from pandas.core.common import isnull, array_equivalent
 import pandas.core.common as com
 from pandas.core.common import (_values_from_object, is_float, is_integer,
@@ -2088,12 +2089,13 @@ def _evaluate_with_datetime_like(self, other, op, opstr):
     def _add_numeric_methods_disabled(cls):
         """ add in numeric methods to disable """
 
-        def _make_invalid_op(opstr):
+        def _make_invalid_op(name):
 
-            def _invalid_op(self, other=None):
-                raise TypeError("cannot perform {opstr} with this index type: {typ}".format(opstr=opstr,
-                                                                                            typ=type(self)))
-            return _invalid_op
+            def invalid_op(self, other=None):
+                raise TypeError("cannot perform {name} with this index type: {typ}".format(name=name,
+                                                                                           typ=type(self)))
+            invalid_op.__name__ = name
+            return invalid_op
 
         cls.__mul__ = cls.__rmul__ = _make_invalid_op('__mul__')
         cls.__floordiv__ = cls.__rfloordiv__ = _make_invalid_op('__floordiv__')
@@ -2178,8 +2180,62 @@ def _evaluate_numeric_unary(self):
         cls.__abs__ = _make_evaluate_unary(lambda x: np.abs(x),'__abs__')
         cls.__inv__ = _make_evaluate_unary(lambda x: -x,'__inv__')
 
+    @classmethod
+    def _add_logical_methods(cls):
+        """ add in logical methods """
+
+        _doc = """
+
+        %(desc)s
+
+        Parameters
+        ----------
+        All arguments to numpy.%(outname)s are accepted.
+
+        Returns
+        -------
+        %(outname)s : bool or array_like (if axis is specified)
+            A single element array_like may be converted to bool."""
+
+        def _make_logical_function(name, desc, f):
+
+            @Substitution(outname=name, desc=desc)
+            @Appender(_doc)
+            def logical_func(self, *args, **kwargs):
+                result = f(self.values)
+                if isinstance(result, (np.ndarray, com.ABCSeries, Index)) \
+                   and result.ndim == 0:
+                    # return NumPy type
+                    return result.dtype.type(result.item())
+                else:  # pragma: no cover
+                    return result
+            logical_func.__name__ = name
+            return logical_func
+
+        cls.all = _make_logical_function(
+            'all', 'Return whether all elements are True', np.all)
+        cls.any = _make_logical_function(
+            'any', 'Return whether any element is True', np.any)
+
+    @classmethod
+    def _add_logical_methods_disabled(cls):
+        """ add in logical methods to disable """
+
+        def _make_invalid_op(name):
+
+            def invalid_op(self, other=None):
+                raise TypeError("cannot perform {name} with this index type: {typ}".format(name=name,
+                                                                                           typ=type(self)))
+            invalid_op.__name__ = name
+            return invalid_op
+
+        cls.all = _make_invalid_op('all')
+        cls.any = _make_invalid_op('any')
+
 
 Index._add_numeric_methods_disabled()
+Index._add_logical_methods()
+
 
 class NumericIndex(Index):
     """
@@ -2291,7 +2347,11 @@ def equals(self, other):
     def _wrap_joined_index(self, joined, other):
         name = self.name if self.name == other.name else None
         return Int64Index(joined, name=name)
+
+
 Int64Index._add_numeric_methods()
+Int64Index._add_logical_methods()
+
 
 class Float64Index(NumericIndex):
 
@@ -2483,7 +2543,10 @@ def isin(self, values, level=None):
             self._validate_index_level(level)
         return lib.ismember_nans(self._array_values(), value_set,
                                  isnull(list(value_set)).any())
+
+
 Float64Index._add_numeric_methods()
+Float64Index._add_logical_methods_disabled()
 
 
 class MultiIndex(Index):
@@ -4436,7 +4499,11 @@ def isin(self, values, level=None):
                 return np.zeros(len(labs), dtype=np.bool_)
             else:
                 return np.lib.arraysetops.in1d(labs, sought_labels)
+
+
 MultiIndex._add_numeric_methods_disabled()
+MultiIndex._add_logical_methods_disabled()
+
 
 # For utility purposes
 

pandas/tests/test_index.py

@@ -75,6 +75,15 @@ def test_numeric_compat(self):
                               "cannot perform __floordiv__",
                               lambda : 1 // idx)
 
+    def test_logical_compat(self):
+        idx = self.create_index()
+        tm.assertRaisesRegexp(TypeError,
+                              'cannot perform all',
+                              lambda : idx.all())
+        tm.assertRaisesRegexp(TypeError,
+                              'cannot perform any',
+                              lambda : idx.any())
+
     def test_boolean_context_compat(self):
 
         # boolean context compat
@@ -820,6 +829,11 @@ def test_take(self):
         expected = self.dateIndex[indexer]
         self.assertTrue(result.equals(expected))
 
+    def test_logical_compat(self):
+        idx = self.create_index()
+        self.assertEqual(idx.all(), idx.values.all())
+        self.assertEqual(idx.any(), idx.values.any())
+
     def _check_method_works(self, method):
         method(self.empty)
         method(self.dateIndex)
@@ -1467,6 +1481,11 @@ def test_equals(self):
         self.assertTrue(self.index.equals(same_values))
         self.assertTrue(same_values.equals(self.index))
 
+    def test_logical_compat(self):
+        idx = self.create_index()
+        self.assertEqual(idx.all(), idx.values.all())
+        self.assertEqual(idx.any(), idx.values.any())
+
     def test_identical(self):
         i = Index(self.index.copy())
         self.assertTrue(i.identical(self.index))

pandas/tests/test_panel.py

@@ -9,6 +9,7 @@
 
 from pandas import Series, DataFrame, Index, isnull, notnull, pivot, MultiIndex
 from pandas.core.datetools import bday
+from pandas.core.nanops import nanall, nanany
 from pandas.core.panel import Panel
 from pandas.core.series import remove_na
 import pandas.core.common as com
@@ -2102,6 +2103,24 @@ def test_update_raise(self):
         np.testing.assert_raises(Exception, pan.update, *(pan,),
                                  **{'raise_conflict': True})
 
+    def test_all_any(self):
+        self.assertTrue((self.panel.all(axis=0).values ==
+                         nanall(self.panel, axis=0)).all())
+        self.assertTrue((self.panel.all(axis=1).values ==
+                         nanall(self.panel, axis=1).T).all())
+        self.assertTrue((self.panel.all(axis=2).values ==
+                         nanall(self.panel, axis=2).T).all())
+        self.assertTrue((self.panel.any(axis=0).values ==
+                         nanany(self.panel, axis=0)).all())
+        self.assertTrue((self.panel.any(axis=1).values ==
+                         nanany(self.panel, axis=1).T).all())
+        self.assertTrue((self.panel.any(axis=2).values ==
+                         nanany(self.panel, axis=2).T).all())
+
+    def test_all_any_unhandled(self):
+        self.assertRaises(NotImplementedError, self.panel.all, bool_only=True)
+        self.assertRaises(NotImplementedError, self.panel.any, bool_only=True)
+
 
 class TestLongPanel(tm.TestCase):
     """