BUG: Fixed union_categoricals with unordered cats (#19097)

* BUG: Fixed union_categoricals with unordered cats

Closes #19096

* TST: Added concat test

Author: TomAugspurger

doc/source/whatsnew/v0.23.0.txt

@@ -453,8 +453,11 @@ Numeric
 Categorical
 ^^^^^^^^^^^
 
-- Bug in ``Categorical.equals`` between two unordered categories with the same categories, but in a different order (:issue:`16603`)
 -
+- Bug in :func:`pandas.api.types.union_categoricals` returning the wrong result
+  when all the categoricals had the same categories, but in a different order.
+  This affected :func:`pandas.concat` with Categorical data (:issue:`19096`).
+- Bug in ``Categorical.equals`` between two unordered categories with the same categories, but in a different order (:issue:`16603`)
 -
 
 Other

pandas/core/dtypes/concat.py

@@ -339,7 +339,16 @@ def _maybe_unwrap(x):
         # identical categories - fastpath
         categories = first.categories
         ordered = first.ordered
-        new_codes = np.concatenate([c.codes for c in to_union])
+
+        if all(first.categories.equals(other.categories)
+               for other in to_union[1:]):
+            new_codes = np.concatenate([c.codes for c in to_union])
+        else:
+            codes = [first.codes] + [_recode_for_categories(other.codes,
+                                                            other.categories,
+                                                            first.categories)
+                                     for other in to_union[1:]]
+            new_codes = np.concatenate(codes)
 
         if sort_categories and not ignore_order and ordered:
             raise TypeError("Cannot use sort_categories=True with "

pandas/tests/reshape/test_concat.py

@@ -481,6 +481,15 @@ def test_concat_categorical(self):
         tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
         tm.assert_series_equal(s1.append(s2, ignore_index=True), exp)
 
+    def test_union_categorical_same_categories_different_order(self):
+        # https://github.com/pandas-dev/pandas/issues/19096
+        a = pd.Series(Categorical(['a', 'b', 'c'], categories=['a', 'b', 'c']))
+        b = pd.Series(Categorical(['a', 'b', 'c'], categories=['b', 'a', 'c']))
+        result = pd.concat([a, b], ignore_index=True)
+        expected = pd.Series(Categorical(['a', 'b', 'c', 'a', 'b', 'c'],
+                                         categories=['a', 'b', 'c']))
+        tm.assert_series_equal(result, expected)
+
     def test_concat_categorical_coercion(self):
         # GH 13524
 

pandas/tests/reshape/test_union_categoricals.py

@@ -129,6 +129,15 @@ def test_union_categorical_same_category(self):
                           categories=['x', 'y', 'z'])
         tm.assert_categorical_equal(res, exp)
 
+    def test_union_categorical_same_categories_different_order(self):
+        # https://github.com/pandas-dev/pandas/issues/19096
+        c1 = Categorical(['a', 'b', 'c'], categories=['a', 'b', 'c'])
+        c2 = Categorical(['a', 'b', 'c'], categories=['b', 'a', 'c'])
+        result = union_categoricals([c1, c2])
+        expected = Categorical(['a', 'b', 'c', 'a', 'b', 'c'],
+                               categories=['a', 'b', 'c'])
+        tm.assert_categorical_equal(result, expected)
+
     def test_union_categoricals_ordered(self):
         c1 = Categorical([1, 2, 3], ordered=True)
         c2 = Categorical([1, 2, 3], ordered=False)