Catch Exception in combine (#22936)

Author: TomAugspurger

pandas/core/arrays/base.py

@@ -739,14 +739,22 @@ def _create_method(cls, op, coerce_to_dtype=True):
         ----------
         op : function
             An operator that takes arguments op(a, b)
-        coerce_to_dtype :  bool
+        coerce_to_dtype :  bool, default True
             boolean indicating whether to attempt to convert
-            the result to the underlying ExtensionArray dtype
-            (default True)
+            the result to the underlying ExtensionArray dtype.
+            If it's not possible to create a new ExtensionArray with the
+            values, an ndarray is returned instead.
 
         Returns
         -------
-        A method that can be bound to a method of a class
+        Callable[[Any, Any], Union[ndarray, ExtensionArray]]
+            A method that can be bound to a class. When used, the method
+            receives the two arguments, one of which is the instance of
+            this class, and should return an ExtensionArray or an ndarray.
+
+            Returning an ndarray may be necessary when the result of the
+            `op` cannot be stored in the ExtensionArray. The dtype of the
+            ndarray uses NumPy's normal inference rules.
 
         Example
         -------
@@ -757,7 +765,6 @@ def _create_method(cls, op, coerce_to_dtype=True):
         in the class definition of MyExtensionArray to create the operator
         for addition, that will be based on the operator implementation
         of the underlying elements of the ExtensionArray
-
         """
 
         def _binop(self, other):
@@ -777,8 +784,13 @@ def convert_values(param):
             if coerce_to_dtype:
                 try:
                     res = self._from_sequence(res)
-                except TypeError:
-                    pass
+                except Exception:
+                    # https://github.com/pandas-dev/pandas/issues/22850
+                    # We catch all regular exceptions here, and fall back
+                    # to an ndarray.
+                    res = np.asarray(res)
+            else:
+                res = np.asarray(res)
 
             return res
 

pandas/core/series.py

@@ -2323,10 +2323,14 @@ def combine(self, other, func, fill_value=None):
             pass
         elif is_extension_array_dtype(self.values):
             # The function can return something of any type, so check
-            # if the type is compatible with the calling EA
+            # if the type is compatible with the calling EA.
             try:
                 new_values = self._values._from_sequence(new_values)
-            except TypeError:
+            except Exception:
+                # https://github.com/pandas-dev/pandas/issues/22850
+                # pandas has no control over what 3rd-party ExtensionArrays
+                # do in _values_from_sequence. We still want ops to work
+                # though, so we catch any regular Exception.
                 pass
 
         return self._constructor(new_values, index=new_index, name=new_name)

pandas/tests/extension/decimal/__init__.py

@@ -0,0 +1,4 @@
+from .array import DecimalArray, DecimalDtype, to_decimal, make_data
+
+
+__all__ = ['DecimalArray', 'DecimalDtype', 'to_decimal', 'make_data']

pandas/tests/extension/decimal/array.py

@@ -1,5 +1,6 @@
 import decimal
 import numbers
+import random
 import sys
 
 import numpy as np
@@ -138,5 +139,13 @@ def _concat_same_type(cls, to_concat):
         return cls(np.concatenate([x._data for x in to_concat]))
 
 
+def to_decimal(values, context=None):
+    return DecimalArray([decimal.Decimal(x) for x in values], context=context)
+
+
+def make_data():
+    return [decimal.Decimal(random.random()) for _ in range(100)]
+
+
 DecimalArray._add_arithmetic_ops()
 DecimalArray._add_comparison_ops()

pandas/tests/extension/decimal/test_decimal.py

@@ -1,18 +1,14 @@
+import operator
 import decimal
 
-import random
 import numpy as np
 import pandas as pd
 import pandas.util.testing as tm
 import pytest
 
 from pandas.tests.extension import base
 
-from .array import DecimalDtype, DecimalArray
-
-
-def make_data():
-    return [decimal.Decimal(random.random()) for _ in range(100)]
+from .array import DecimalDtype, DecimalArray, make_data
 
 
 @pytest.fixture
@@ -275,3 +271,47 @@ def test_compare_array(self, data, all_compare_operators):
         other = pd.Series(data) * [decimal.Decimal(pow(2.0, i))
                                    for i in alter]
         self._compare_other(s, data, op_name, other)
+
+
+class DecimalArrayWithoutFromSequence(DecimalArray):
+    """Helper class for testing error handling in _from_sequence."""
+    def _from_sequence(cls, scalars, dtype=None, copy=False):
+        raise KeyError("For the test")
+
+
+class DecimalArrayWithoutCoercion(DecimalArrayWithoutFromSequence):
+    @classmethod
+    def _create_arithmetic_method(cls, op):
+        return cls._create_method(op, coerce_to_dtype=False)
+
+
+DecimalArrayWithoutCoercion._add_arithmetic_ops()
+
+
+def test_combine_from_sequence_raises():
+    # https://github.com/pandas-dev/pandas/issues/22850
+    ser = pd.Series(DecimalArrayWithoutFromSequence([
+        decimal.Decimal("1.0"),
+        decimal.Decimal("2.0")
+    ]))
+    result = ser.combine(ser, operator.add)
+
+    # note: object dtype
+    expected = pd.Series([decimal.Decimal("2.0"),
+                          decimal.Decimal("4.0")], dtype="object")
+    tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("class_", [DecimalArrayWithoutFromSequence,
+                                    DecimalArrayWithoutCoercion])
+def test_scalar_ops_from_sequence_raises(class_):
+    # op(EA, EA) should return an EA, or an ndarray if it's not possible
+    # to return an EA with the return values.
+    arr = class_([
+        decimal.Decimal("1.0"),
+        decimal.Decimal("2.0")
+    ])
+    result = arr + arr
+    expected = np.array([decimal.Decimal("2.0"), decimal.Decimal("4.0")],
+                        dtype="object")
+    tm.assert_numpy_array_equal(result, expected)

pandas/tests/extension/json/__init__.py

@@ -0,0 +1,3 @@
+from .array import JSONArray, JSONDtype, make_data
+
+__all__ = ['JSONArray', 'JSONDtype', 'make_data']

pandas/tests/extension/json/array.py

@@ -13,6 +13,8 @@
 import collections
 import itertools
 import numbers
+import random
+import string
 import sys
 
 import numpy as np
@@ -179,3 +181,10 @@ def _values_for_argsort(self):
         # cast them to an (N, P) array, instead of an (N,) array of tuples.
         frozen = [()] + [tuple(x.items()) for x in self]
         return np.array(frozen, dtype=object)[1:]
+
+
+def make_data():
+    # TODO: Use a regular dict. See _NDFrameIndexer._setitem_with_indexer
+    return [collections.UserDict([
+        (random.choice(string.ascii_letters), random.randint(0, 100))
+        for _ in range(random.randint(0, 10))]) for _ in range(100)]

pandas/tests/extension/json/test_json.py

@@ -1,7 +1,5 @@
 import operator
 import collections
-import random
-import string
 
 import pytest
 
@@ -10,18 +8,11 @@
 from pandas.compat import PY2, PY36
 from pandas.tests.extension import base
 
-from .array import JSONArray, JSONDtype
+from .array import JSONArray, JSONDtype, make_data
 
 pytestmark = pytest.mark.skipif(PY2, reason="Py2 doesn't have a UserDict")
 
 
-def make_data():
-    # TODO: Use a regular dict. See _NDFrameIndexer._setitem_with_indexer
-    return [collections.UserDict([
-        (random.choice(string.ascii_letters), random.randint(0, 100))
-        for _ in range(random.randint(0, 10))]) for _ in range(100)]
-
-
 @pytest.fixture
 def dtype():
     return JSONDtype()