Merge branch 'index-values' into pandas-array-upstream+fu1

Author: TomAugspurger

pandas/core/arrays/base.py

@@ -312,3 +312,15 @@ def value_counts(self, dropna=True):
             self = self[~self.isna()]
 
         return value_counts(np.array(self))
+
+    @property
+    def _ndarray_values(self):
+        # type: () -> np.ndarray
+        """Internal pandas method for lossy conversion to a NumPy ndarray.
+
+        This method is not part of the pandas interface.
+
+        The expectation is that this is cheap to compute, and is primarily
+        used for interacting with our indexers.
+        """
+        return np.array(self)

pandas/core/arrays/categorical.py

@@ -410,6 +410,10 @@ def dtype(self):
         """The :class:`~pandas.api.types.CategoricalDtype` for this instance"""
         return self._dtype
 
+    @property
+    def _ndarray_values(self):
+        return self.codes
+
     @property
     def _constructor(self):
         return Categorical

pandas/core/base.py

@@ -14,7 +14,8 @@
     is_scalar,
     is_datetimelike,
     is_categorical_dtype,
-    is_extension_type)
+    is_extension_type,
+    is_extension_array_dtype)
 
 from pandas.util._validators import validate_bool_kwarg
 
@@ -772,18 +773,14 @@ def base(self):
     def _ndarray_values(self):
         """The data as an ndarray, possibly losing information.
 
-        The expectation is that this is cheap to compute.
+        The expectation is that this is cheap to compute, and is primarily
+        used for interacting with our indexers.
 
         - categorical -> codes
-
-        See '_values' for more.
         """
         # type: () -> np.ndarray
-        from pandas.core.dtypes.common import is_categorical_dtype
-
-        if is_categorical_dtype(self):
-            return self._values.codes
-
+        if is_extension_array_dtype(self):
+            return self.values._ndarray_values
         return self.values
 
     @property
@@ -833,10 +830,8 @@ def tolist(self):
 
         if is_datetimelike(self):
             return [com._maybe_box_datetimelike(x) for x in self._values]
-        elif is_categorical_dtype(self):
-            return self.values.tolist()
         else:
-            return self._ndarray_values.tolist()
+            return self._values.tolist()
 
     def __iter__(self):
         """

pandas/core/indexes/category.py

@@ -293,10 +293,6 @@ def values(self):
         """ return the underlying data, which is a Categorical """
         return self._data
 
-    @property
-    def _ndarray_values(self):
-        return self._data.codes
-
     @property
     def itemsize(self):
         return self.values.itemsize

pandas/tests/indexes/datetimes/test_datetime.py

@@ -469,3 +469,18 @@ def test_factorize_dst(self):
             arr, res = obj.factorize()
             tm.assert_numpy_array_equal(arr, np.arange(12, dtype=np.intp))
             tm.assert_index_equal(res, idx)
+
+    @pytest.mark.parametrize('arr, expected', [
+        (pd.DatetimeIndex(['2017', '2017']), pd.DatetimeIndex(['2017'])),
+        (pd.DatetimeIndex(['2017', '2017'], tz='US/Eastern'),
+         pd.DatetimeIndex(['2017'], tz='US/Eastern')),
+    ])
+    def test_unique(self, arr, expected):
+        result = arr.unique()
+
+        if isinstance(expected, np.ndarray):
+            tm.assert_numpy_array_equal(result, expected)
+        if isinstance(expected, pd.Series):
+            tm.assert_series_equal(result, expected)
+        if isinstance(expected, pd.DatetimeIndex):
+            tm.assert_index_equal(result, expected)

pandas/tests/indexes/test_multi.py

@@ -962,6 +962,53 @@ def test_values_boxed(self):
         # Check that code branches for boxed values produce identical results
         tm.assert_numpy_array_equal(result.values[:4], result[:4].values)
 
+    def test_values_multiindex_datetimeindex(self):
+        # Test to ensure we hit the boxing / nobox part of MI.values
+        ints = np.arange(10**18, 10**18 + 5)
+        naive = pd.DatetimeIndex(ints)
+        aware = pd.DatetimeIndex(ints, tz='US/Central')
+
+        idx = pd.MultiIndex.from_arrays([naive, aware])
+        result = idx.values
+
+        outer = pd.DatetimeIndex([x[0] for x in result])
+        tm.assert_index_equal(outer, naive)
+
+        inner = pd.DatetimeIndex([x[1] for x in result])
+        tm.assert_index_equal(inner, aware)
+
+        # n_lev > n_lab
+        result = idx[:2].values
+
+        outer = pd.DatetimeIndex([x[0] for x in result])
+        tm.assert_index_equal(outer, naive[:2])
+
+        inner = pd.DatetimeIndex([x[1] for x in result])
+        tm.assert_index_equal(inner, aware[:2])
+
+    def test_values_multiindex_periodindex(self):
+        # Test to ensure we hit the boxing / nobox part of MI.values
+        ints = np.arange(2007, 2012)
+        pidx = pd.PeriodIndex(ints, freq='D')
+
+        idx = pd.MultiIndex.from_arrays([ints, pidx])
+        result = idx.values
+
+        outer = pd.Int64Index([x[0] for x in result])
+        tm.assert_index_equal(outer, pd.Int64Index(ints))
+
+        inner = pd.PeriodIndex([x[1] for x in result])
+        tm.assert_index_equal(inner, pidx)
+
+        # n_lev > n_lab
+        result = idx[:2].values
+
+        outer = pd.Int64Index([x[0] for x in result])
+        tm.assert_index_equal(outer, pd.Int64Index(ints[:2]))
+
+        inner = pd.PeriodIndex([x[1] for x in result])
+        tm.assert_index_equal(inner, pidx[:2])
+
     def test_append(self):
         result = self.index[:3].append(self.index[3:])
         assert result.equals(self.index)

pandas/tests/test_base.py

@@ -1178,39 +1178,6 @@ def test_iter_box(self):
             assert res == exp
 
 
-@pytest.mark.parametrize('arr, expected', [
-    (pd.DatetimeIndex(['2017', '2017']), pd.DatetimeIndex(['2017'])),
-    (pd.DatetimeIndex(['2017', '2017'], tz='US/Eastern'),
-     pd.DatetimeIndex(['2017'], tz='US/Eastern')),
-])
-def test_unique_datetime_index(arr, expected):
-    result = arr.unique()
-
-    if isinstance(expected, np.ndarray):
-        tm.assert_numpy_array_equal(result, expected)
-    if isinstance(expected, pd.Series):
-        tm.assert_series_equal(result, expected)
-    if isinstance(expected, pd.DatetimeIndex):
-        tm.assert_index_equal(result, expected)
-
-
-@pytest.mark.parametrize('arr, expected', [
-    (pd.Series(pd.DatetimeIndex(['2017', '2017'])),
-     np.array(['2017-01-01T00:00:00'], dtype='M8[ns]')),
-    (pd.Series(pd.DatetimeIndex(['2017', '2017'], tz='US/Eastern')),
-     np.array([pd.Timestamp('2017', tz="US/Eastern")], dtype=object)),
-])
-def test_unique_datetime_series(arr, expected):
-    result = arr.unique()
-
-    if isinstance(expected, np.ndarray):
-        tm.assert_numpy_array_equal(result, expected)
-    if isinstance(expected, pd.Series):
-        tm.assert_series_equal(result, expected)
-    if isinstance(expected, pd.DatetimeIndex):
-        tm.assert_index_equal(result, expected)
-
-
 @pytest.mark.parametrize('array, expected_type, dtype', [
     (np.array([0, 1], dtype=np.int64), np.ndarray, 'int64'),
     (np.array(['a', 'b']), np.ndarray, 'object'),
@@ -1260,52 +1227,3 @@ def test_ndarray_values(array, expected):
     r_values = pd.Index(array)._ndarray_values
     tm.assert_numpy_array_equal(l_values, r_values)
     tm.assert_numpy_array_equal(l_values, expected)
-
-
-def test_values_multiindex_datetimeindex():
-    # Test to ensure we hit the boxing / nobox part of MI.values
-    ints = np.arange(10**18, 10**18 + 5)
-    naive = pd.DatetimeIndex(ints)
-    aware = pd.DatetimeIndex(ints, tz='US/Central')
-
-    idx = pd.MultiIndex.from_arrays([naive, aware])
-    result = idx.values
-
-    outer = pd.DatetimeIndex([x[0] for x in result])
-    tm.assert_index_equal(outer, naive)
-
-    inner = pd.DatetimeIndex([x[1] for x in result])
-    tm.assert_index_equal(inner, aware)
-
-    # n_lev > n_lab
-    result = idx[:2].values
-
-    outer = pd.DatetimeIndex([x[0] for x in result])
-    tm.assert_index_equal(outer, naive[:2])
-
-    inner = pd.DatetimeIndex([x[1] for x in result])
-    tm.assert_index_equal(inner, aware[:2])
-
-
-def test_values_multiindex_periodindex():
-    # Test to ensure we hit the boxing / nobox part of MI.values
-    ints = np.arange(2007, 2012)
-    pidx = pd.PeriodIndex(ints, freq='D')
-
-    idx = pd.MultiIndex.from_arrays([ints, pidx])
-    result = idx.values
-
-    outer = pd.Int64Index([x[0] for x in result])
-    tm.assert_index_equal(outer, pd.Int64Index(ints))
-
-    inner = pd.PeriodIndex([x[1] for x in result])
-    tm.assert_index_equal(inner, pidx)
-
-    # n_lev > n_lab
-    result = idx[:2].values
-
-    outer = pd.Int64Index([x[0] for x in result])
-    tm.assert_index_equal(outer, pd.Int64Index(ints[:2]))
-
-    inner = pd.PeriodIndex([x[1] for x in result])
-    tm.assert_index_equal(inner, pidx[:2])