changed according to comments

Author: topper-123

pandas/_libs/algos.pyx

@@ -10,11 +10,13 @@ from libc.math cimport fabs, sqrt
 import numpy as np
 cimport numpy as cnp
 from numpy cimport (ndarray,
-                    NPY_INT64, NPY_UINT64, NPY_INT32, NPY_INT16, NPY_INT8,
-                    NPY_FLOAT32, NPY_FLOAT64,
+                    NPY_INT64, NPY_INT32, NPY_INT16, NPY_INT8,
+                    NPY_UINT64, NPY_UINT32, NPY_UINT16, NPY_UINT8,
+                    NPY_FLOAT64, NPY_FLOAT32,
                     NPY_OBJECT,
-                    int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t,
-                    uint32_t, uint64_t, float32_t, float64_t,
+                    int64_t, int32_t, int16_t, int8_t,
+                    uint64_t, uint32_t, uint16_t, uint8_t,
+                    float64_t, float32_t,
                     double_t)
 cnp.import_array()
 

pandas/_libs/algos_common_helper.pxi.in

@@ -133,11 +133,14 @@ def ensure_object(object arr):
 # name, c_type, dtype
 dtypes = [('float64', 'FLOAT64', 'float64'),
           ('float32', 'FLOAT32', 'float32'),
-          ('int8', 'INT8', 'int8'),
-          ('int16', 'INT16', 'int16'),
-          ('int32', 'INT32', 'int32'),
           ('int64', 'INT64', 'int64'),
+          ('int32', 'INT32', 'int32'),
+          ('int16', 'INT16', 'int16'),
+          ('int8', 'INT8', 'int8'),
           ('uint64', 'UINT64', 'uint64'),
+          ('uint32', 'UINT32', 'uint32'),
+          ('uint16', 'UINT16', 'uint16'),
+          ('uint8', 'UINT8', 'uint8'),
           # ('platform_int', 'INT', 'int_'),
           # ('object', 'OBJECT', 'object_'),
 ]

pandas/_libs/index.pyx

@@ -7,7 +7,7 @@ import numpy as np
 cimport numpy as cnp
 from numpy cimport (ndarray,
                     float64_t, float32_t,
-                    int64_t,int32_t, int16_t, int8_t,
+                    int64_t, int32_t, int16_t, int8_t,
                     uint64_t, uint32_t, uint16_t, uint8_t,
                     intp_t,
                     # Note: NPY_DATETIME, NPY_TIMEDELTA are only available

pandas/_libs/index_class_helper.pxi.in

@@ -30,6 +30,8 @@ dtypes = [('Float64', 'float64', 'float64_t'),
 
 cdef class {{name}}Engine(IndexEngine):
 
+    _dtype = '{{dtype}}'
+
     def _call_monotonic(self, values):
         return algos.is_monotonic_{{dtype}}(values, timelike=False)
 
@@ -61,6 +63,14 @@ cdef class {{name}}Engine(IndexEngine):
     cpdef _call_map_locations(self, values):
         # self.mapping is of type Int64HashTable, so convert dtype of values
         self.mapping.map_locations(algos.ensure_int64(values))
+    {{elif name in {'UInt8', 'UInt16', 'UInt32'} }}
+    cpdef _call_map_locations(self, values):
+        # self.mapping is of type UInt64HashTable, so convert dtype of values
+        self.mapping.map_locations(algos.ensure_uint64(values))
+    {{elif name in {'Float32'} }}
+    cpdef _call_map_locations(self, values):
+        # self.mapping is of type Float64HashTable, so convert dtype of values
+        self.mapping.map_locations(algos.ensure_float64(values))
     {{endif}}
 
     {{if name != 'Float64' and name != 'Object'}}
@@ -83,7 +93,7 @@ cdef class {{name}}Engine(IndexEngine):
             ndarray[{{ctype}}] values
             int count = 0
 
-        {{if name != 'Float64'}}
+        {{if name not in {'Float64', 'Float32'} }}
         if not util.is_integer_object(val):
             raise KeyError(val)
         {{endif}}

pandas/conftest.py

@@ -102,14 +102,29 @@ def ip():
 
 
 @pytest.fixture(params=[True, False, None])
-def observed(request):
+def _true_false_none(request):
+    """
+    Base fixture for fixtures that return True, False and None.
+    """
+    return request.param
+
+
+@pytest.fixture
+def observed(_true_false_none):
     """ pass in the observed keyword to groupby for [True, False]
     This indicates whether categoricals should return values for
     values which are not in the grouper [False / None], or only values which
     appear in the grouper [True]. [None] is supported for future compatiblity
     if we decide to change the default (and would need to warn if this
     parameter is not passed)"""
-    return request.param
+    return _true_false_none
+
+
+@pytest.fixture
+def ordered(_true_false_none):
+    """Return the allowed parameters for Categorical/CategoricalIndex.ordered.
+    """
+    return _true_false_none
 
 
 _all_arithmetic_operators = ['__add__', '__radd__',

pandas/tests/indexes/conftest.py

@@ -4,6 +4,7 @@
 
 import pandas.util.testing as tm
 from pandas.core.indexes.api import Index, MultiIndex
+from pandas._libs import index as li
 from pandas.compat import lzip, long
 
 
@@ -45,3 +46,15 @@ def zero(request):
     # For testing division by (or of) zero for Index with length 5, this
     # gives several scalar-zeros and length-5 vector-zeros
     return request.param
+
+
+@pytest.fixture(
+    params=[
+        'Int64', 'Int32', 'Int16', 'Int8',
+        'UInt64', 'UInt32', 'UInt16', 'UInt8',
+        'Float64', 'Float32',
+    ])
+def num_engine(request):
+    """Return the various numeric engines in pd._libs.index
+    """
+    return getattr(li, "{}Engine".format(request.param))

pandas/tests/indexes/test_category.py

@@ -14,6 +14,7 @@
 from pandas import Categorical, IntervalIndex, compat
 from pandas.util.testing import assert_almost_equal
 import pandas.core.config as cf
+from pandas._libs import index as li
 import pandas as pd
 
 if PY3:
@@ -1117,3 +1118,27 @@ def test_take_invalid_kwargs(self):
         msg = "the 'mode' parameter is not supported"
         tm.assert_raises_regex(ValueError, msg, idx.take,
                                indices, mode='clip')
+
+
+class TestCategoricalIndexEngine(object):
+
+    @pytest.mark.parametrize('nbits', [8, 16, 32, 64])
+    def test_engine_type(self, nbits):
+        """Check that a CategoricalIndex has the correct engine type.
+        """
+        if nbits < 64:
+            ncategories = int(2 ** (nbits / 2) / 2)  # 128 for nbits==16 etc
+            index = CategoricalIndex(range(ncategories))
+        else:
+            index = CategoricalIndex(['a', 'b', 'c'])
+            # having actual 2 ** (64 / 2) / 2 categories is too
+            # memory-intensive, so we set codes.dtype manually
+            index._values._codes = index._values._codes.astype('int64')
+
+        dtype = {8: np.int8, 16: np.int16,
+                 32: np.int32, 64: np.int64}[nbits]
+        engine = {8: li.Int8Engine, 16: li.Int16Engine,
+                  32: li.Int32Engine, 64: li.Int64Engine}[nbits]
+
+        assert isinstance(index._engine, engine)
+        assert issubclass(index.codes.dtype.type, dtype)

pandas/tests/indexes/test_engine.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas._libs.index import (Int64Engine, UInt64Engine,
+                                Float64Engine, ObjectEngine)
+
+
+class TestNumericEngine(object):
+
+    @pytest.mark.parametrize('data', [[0, 1, 2]])
+    def test_engine_type(self, data, num_engine):
+        index = pd.Index(data, dtype=num_engine._dtype)
+        if issubclass(index.dtype.type, np.signedinteger):
+            assert isinstance(index._engine, Int64Engine)
+        elif issubclass(index.dtype.type, np.unsignedinteger):
+            assert isinstance(index._engine, UInt64Engine)
+        elif issubclass(index.dtype.type, np.floating):
+            assert isinstance(index._engine, Float64Engine)
+        else:
+            raise TypeError("unexpected dtype {}".format(index.dtype))
+
+    @pytest.mark.parametrize('data', [[0, 1, 2]])
+    def test_is_monotonic_ordered(self, data, num_engine):
+        codes = np.array(data, dtype=num_engine._dtype)
+        e = num_engine(lambda: codes, len(codes))
+        assert e.is_monotonic_increasing
+        assert not e.is_monotonic_decreasing
+
+        # reverse sort order
+        codes = np.array(list(reversed(data)), dtype=num_engine._dtype)
+        e = num_engine(lambda: codes, len(codes))
+        assert not e.is_monotonic_increasing
+        assert e.is_monotonic_decreasing
+
+    @pytest.mark.parametrize('data', [[1, 0, 2]])
+    def test_is_not_monotonic_ordered(self, data, num_engine):
+        codes = np.array(data, dtype=num_engine._dtype)
+        e = num_engine(lambda: codes, len(codes))
+        assert not e.is_monotonic_increasing
+        assert not e.is_monotonic_decreasing
+
+    @pytest.mark.parametrize('values, expected', [
+        ([1, 2, 3], True),
+        ([1, 1, 2], False),
+    ])
+    def test_is_unique(self, values, expected, num_engine):
+
+        codes = np.array(values, dtype=num_engine._dtype)
+        e = num_engine(lambda: codes, len(codes))
+        assert e.is_unique is expected
+
+
+class TestObjectEngine(object):
+
+    def setup_class(cls):
+        cls.Engine = ObjectEngine
+        cls.dtype = object
+
+    @pytest.mark.parametrize('data', [['a', 'b', 'c']])
+    def test_engine_type(self, data):
+        index = pd.Index(data)
+        assert isinstance(index._engine, self.Engine)
+
+    @pytest.mark.parametrize('data', [['a', 'b', 'c']])
+    def test_is_monotonic_ordered(self, data):
+        codes = np.array(data, dtype=self.dtype)
+        e = self.Engine(lambda: codes, len(codes))
+        assert e.is_monotonic_increasing
+        assert not e.is_monotonic_decreasing
+
+        # reverse sort order
+        codes = np.array(list(reversed(data)), dtype=self.dtype)
+        e = self.Engine(lambda: codes, len(codes))
+        assert not e.is_monotonic_increasing
+        assert e.is_monotonic_decreasing
+
+    @pytest.mark.parametrize('data', [['a', 'c', 'b']])
+    def test_is_not_monotonic_ordered(self, data):
+        codes = np.array(data, dtype=self.dtype)
+        e = self.Engine(lambda: codes, len(codes))
+        assert not e.is_monotonic_increasing
+        assert not e.is_monotonic_decreasing
+
+    @pytest.mark.parametrize('values, expected', [
+        (['a', 'b', 'c'], True),
+        (['a', 'a', 'b'], False),
+    ])
+    def test_is_unique(self, values, expected):
+        codes = np.array(values, dtype=self.dtype)
+        e = self.Engine(lambda: codes, len(codes))
+        assert e.is_unique is expected