ENH: add in extension dtype registry

Author: jreback

doc/source/whatsnew/v0.24.0.txt

@@ -14,14 +14,22 @@ Other Enhancements
 ^^^^^^^^^^^^^^^^^^
 -
 -
--
 
 .. _whatsnew_0240.api_breaking:
 
 
 Backwards incompatible API changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+.. _whatsnew_0240.api.extension:
+
+ExtensionType Changes
+^^^^^^^^^^^^^^^^^^^^^
+
+- ``ExtensionArray`` has gained the abstract methods ``.dropna()`` and ``.append()``, and attribute ``array_type`` (:issue:`21185`)
+- ``ExtensionDtype`` has gained the ability to instantiate from string dtypes, e.g. ``decimal`` would instaniate a registered ``DecimalDtype`` (:issue:`21185`)
+- The ``ExtensionArray`` constructor, ``_from_sequence`` now take the keyword arg ``copy=False`` (:issue:`21185`)
+
 .. _whatsnew_0240.api.other:
 
 Other API Changes
@@ -177,4 +185,3 @@ Other
 -
 -
 -
-

pandas/core/algorithms.py

@@ -154,7 +154,7 @@ def _reconstruct_data(values, dtype, original):
     """
     from pandas import Index
     if is_extension_array_dtype(dtype):
-        pass
+        values = dtype.array_type._from_sequence(values)
     elif is_datetime64tz_dtype(dtype) or is_period_dtype(dtype):
         values = Index(original)._shallow_copy(values, name=None)
     elif is_bool_dtype(dtype):
@@ -705,7 +705,7 @@ def value_counts(values, sort=True, ascending=False, normalize=False,
 
     else:
 
-        if is_categorical_dtype(values) or is_sparse(values):
+        if is_extension_array_dtype(values) or is_sparse(values):
 
             # handle Categorical and sparse,
             result = Series(values)._values.value_counts(dropna=dropna)

pandas/core/arrays/base.py

@@ -36,7 +36,9 @@ class ExtensionArray(object):
     * isna
     * take
     * copy
+    * append
     * _concat_same_type
+    * array_type
 
     An additional method is available to satisfy pandas' internal,
     private block API.
@@ -49,6 +51,7 @@ class ExtensionArray(object):
     methods:
 
     * fillna
+    * dropna
     * unique
     * factorize / _values_for_factorize
     * argsort / _values_for_argsort
@@ -82,14 +85,16 @@ class ExtensionArray(object):
     # Constructors
     # ------------------------------------------------------------------------
     @classmethod
-    def _from_sequence(cls, scalars):
+    def _from_sequence(cls, scalars, copy=False):
         """Construct a new ExtensionArray from a sequence of scalars.
 
         Parameters
         ----------
         scalars : Sequence
             Each element will be an instance of the scalar type for this
             array, ``cls.dtype.type``.
+        copy : boolean, default True
+            if True, copy the underlying data
         Returns
         -------
         ExtensionArray
@@ -379,6 +384,16 @@ def fillna(self, value=None, method=None, limit=None):
             new_values = self.copy()
         return new_values
 
+    def dropna(self):
+        """ Return ExtensionArray without NA values
+
+        Returns
+        -------
+        valid : ExtensionArray
+        """
+
+        return self[~self.isna()]
+
     def unique(self):
         """Compute the ExtensionArray of unique values.
 
@@ -567,6 +582,34 @@ def copy(self, deep=False):
         """
         raise AbstractMethodError(self)
 
+    def append(self, other):
+        """
+        Append a collection of Arrays together
+
+        Parameters
+        ----------
+        other : ExtensionArray or list/tuple of ExtensionArrays
+
+        Returns
+        -------
+        appended : ExtensionArray
+        """
+
+        to_concat = [self]
+        cls = self.__class__
+
+        if isinstance(other, (list, tuple)):
+            to_concat = to_concat + list(other)
+        else:
+            to_concat.append(other)
+
+        for obj in to_concat:
+            if not isinstance(obj, cls):
+                raise TypeError('all inputs must be of type {}'.format(
+                    cls.__name__))
+
+        return cls._concat_same_type(to_concat)
+
     # ------------------------------------------------------------------------
     # Block-related methods
     # ------------------------------------------------------------------------

pandas/core/arrays/categorical.py

@@ -2343,6 +2343,10 @@ def isin(self, values):
         return algorithms.isin(self.codes, code_values)
 
 
+# inform the Dtype about us
+CategoricalDtype.array_type = Categorical
+
+
 # The Series.cat accessor
 
 

pandas/core/dtypes/base.py

@@ -156,6 +156,12 @@ def name(self):
         """
         raise AbstractMethodError(self)
 
+    @property
+    def array_type(self):
+        """Return the array type associated with this dtype
+        """
+        raise AbstractMethodError(self)
+
     @classmethod
     def construct_from_string(cls, string):
         """Attempt to construct this type from a string.

pandas/core/dtypes/cast.py

@@ -647,6 +647,11 @@ def conv(r, dtype):
 def astype_nansafe(arr, dtype, copy=True):
     """ return a view if copy is False, but
         need to be very careful as the result shape could change! """
+
+    # dispatch on extension dtype if needed
+    if is_extension_array_dtype(dtype):
+        return dtype.array_type._from_sequence(arr, copy=copy)
+
     if not isinstance(dtype, np.dtype):
         dtype = pandas_dtype(dtype)
 

pandas/core/dtypes/common.py

@@ -9,7 +9,7 @@
                      DatetimeTZDtype, DatetimeTZDtypeType,
                      PeriodDtype, PeriodDtypeType,
                      IntervalDtype, IntervalDtypeType,
-                     ExtensionDtype, PandasExtensionDtype)
+                     ExtensionDtype, registry)
 from .generic import (ABCCategorical, ABCPeriodIndex,
                       ABCDatetimeIndex, ABCSeries,
                       ABCSparseArray, ABCSparseSeries, ABCCategoricalIndex,
@@ -1975,38 +1975,13 @@ def pandas_dtype(dtype):
     np.dtype or a pandas dtype
     """
 
-    if isinstance(dtype, DatetimeTZDtype):
-        return dtype
-    elif isinstance(dtype, PeriodDtype):
-        return dtype
-    elif isinstance(dtype, CategoricalDtype):
-        return dtype
-    elif isinstance(dtype, IntervalDtype):
-        return dtype
-    elif isinstance(dtype, string_types):
-        try:
-            return DatetimeTZDtype.construct_from_string(dtype)
-        except TypeError:
-            pass
-
-        if dtype.startswith('period[') or dtype.startswith('Period['):
-            # do not parse string like U as period[U]
-            try:
-                return PeriodDtype.construct_from_string(dtype)
-            except TypeError:
-                pass
-
-        elif dtype.startswith('interval') or dtype.startswith('Interval'):
-            try:
-                return IntervalDtype.construct_from_string(dtype)
-            except TypeError:
-                pass
+    # registered extension types
+    result = registry.find(dtype)
+    if result is not None:
+        return result
 
-        try:
-            return CategoricalDtype.construct_from_string(dtype)
-        except TypeError:
-            pass
-    elif isinstance(dtype, (PandasExtensionDtype, ExtensionDtype)):
+    # un-registered extension types
+    if isinstance(dtype, ExtensionDtype):
         return dtype
 
     try:

pandas/core/dtypes/dtypes.py

@@ -2,12 +2,70 @@
 
 import re
 import numpy as np
+from collections import OrderedDict
 from pandas import compat
 from pandas.core.dtypes.generic import ABCIndexClass, ABCCategoricalIndex
 
 from .base import ExtensionDtype, _DtypeOpsMixin
 
 
+class Registry(object):
+    """ Registry for dtype inference
+
+    We can directly construct dtypes in pandas_dtypes if they are
+    a type; the registry allows us to register an extension dtype
+    to try inference from a string or a dtype class
+
+    These are tried in order for inference.
+    """
+    dtypes = OrderedDict()
+
+    @classmethod
+    def register(self, dtype, constructor=None):
+        """
+        Parameters
+        ----------
+        dtype : PandasExtension Dtype
+        """
+        if not issubclass(dtype, (PandasExtensionDtype, ExtensionDtype)):
+            raise ValueError("can only register pandas extension dtypes")
+
+        if constructor is None:
+            constructor = dtype.construct_from_string
+
+        self.dtypes[dtype] = constructor
+
+    def find(self, dtype):
+        """
+        Parameters
+        ----------
+        dtype : PandasExtensionDtype or string
+
+        Returns
+        -------
+        return the first matching dtype, otherwise return None
+        """
+        if not isinstance(dtype, compat.string_types):
+            dtype_type = dtype
+            if not isinstance(dtype, type):
+                dtype_type = type(dtype)
+            if issubclass(dtype_type, (PandasExtensionDtype, ExtensionDtype)):
+                return dtype
+
+            return None
+
+        for dtype_type, constructor in self.dtypes.items():
+            try:
+                return constructor(dtype)
+            except TypeError:
+                pass
+
+        return None
+
+
+registry = Registry()
+
+
 class PandasExtensionDtype(_DtypeOpsMixin):
     """
     A np.dtype duck-typed class, suitable for holding a custom dtype.
@@ -564,6 +622,17 @@ def construct_from_string(cls, string):
                 pass
         raise TypeError("could not construct PeriodDtype")
 
+    @classmethod
+    def construct_from_string_strict(cls, string):
+        """
+        Strict construction from a string, raise a TypeError if not
+        possible
+        """
+        if string.startswith('period[') or string.startswith('Period['):
+            # do not parse string like U as period[U]
+            return PeriodDtype.construct_from_string(string)
+        raise TypeError("could not construct PeriodDtype")
+
     def __unicode__(self):
         return "period[{freq}]".format(freq=self.freq.freqstr)
 
@@ -683,6 +752,16 @@ def construct_from_string(cls, string):
         msg = "a string needs to be passed, got type {typ}"
         raise TypeError(msg.format(typ=type(string)))
 
+    @classmethod
+    def construct_from_string_strict(cls, string):
+        """
+        Strict construction from a string, raise a TypeError if not
+        possible
+        """
+        if string.startswith('interval') or string.startswith('Interval'):
+            return IntervalDtype.construct_from_string(string)
+        raise TypeError("cannot construct IntervalDtype")
+
     def __unicode__(self):
         if self.subtype is None:
             return "interval"
@@ -723,3 +802,10 @@ def is_dtype(cls, dtype):
             else:
                 return False
         return super(IntervalDtype, cls).is_dtype(dtype)
+
+
+# register the dtypes in search order
+registry.register(DatetimeTZDtype)
+registry.register(PeriodDtype, PeriodDtype.construct_from_string_strict)
+registry.register(IntervalDtype, IntervalDtype.construct_from_string_strict)
+registry.register(CategoricalDtype)

pandas/core/internals.py

@@ -633,8 +633,9 @@ def _astype(self, dtype, copy=False, errors='raise', values=None,
             return self.make_block(Categorical(self.values, dtype=dtype))
 
         # astype processing
-        dtype = np.dtype(dtype)
-        if self.dtype == dtype:
+        if not is_extension_array_dtype(dtype):
+            dtype = np.dtype(dtype)
+        if is_dtype_equal(self.dtype, dtype):
             if copy:
                 return self.copy()
             return self
@@ -662,7 +663,13 @@ def _astype(self, dtype, copy=False, errors='raise', values=None,
 
                 # _astype_nansafe works fine with 1-d only
                 values = astype_nansafe(values.ravel(), dtype, copy=True)
-                values = values.reshape(self.shape)
+
+                # TODO(extension)
+                # should we make this attribute?
+                try:
+                    values = values.reshape(self.shape)
+                except AttributeError:
+                    pass
 
             newb = make_block(values, placement=self.mgr_locs,
                               klass=klass)
@@ -3170,6 +3177,10 @@ def get_block_type(values, dtype=None):
         cls = TimeDeltaBlock
     elif issubclass(vtype, np.complexfloating):
         cls = ComplexBlock
+    elif is_categorical(values):
+        cls = CategoricalBlock
+    elif is_extension_array_dtype(values):
+        cls = ExtensionBlock
     elif issubclass(vtype, np.datetime64):
         assert not is_datetimetz(values)
         cls = DatetimeBlock
@@ -3179,10 +3190,6 @@ def get_block_type(values, dtype=None):
         cls = IntBlock
     elif dtype == np.bool_:
         cls = BoolBlock
-    elif is_categorical(values):
-        cls = CategoricalBlock
-    elif is_extension_array_dtype(values):
-        cls = ExtensionBlock
     else:
         cls = ObjectBlock
     return cls