BUG: df.sort_values() not respecting na_position with categoricals #22556

doc/source/whatsnew/v0.24.0.txt

@@ -613,6 +613,7 @@ Categorical
 ^^^^^^^^^^^
 
 - Bug in :meth:`Categorical.from_codes` where ``NaN`` values in ``codes`` were silently converted to ``0`` (:issue:`21767`). In the future this will raise a ``ValueError``. Also changes the behavior of ``.from_codes([1.1, 2.0])``.
+- Bug in :meth:`Categorical.sort_values` where ``NaN`` values were always positioned in front regardless of ``na_position`` value. (:issue:`22556`). `test_stata.py` was incorrectly passing using default ``na_position='last'``.
 
 Datetimelike
 ^^^^^^^^^^^^

pandas/core/sorting.py

@@ -241,7 +241,18 @@ def nargsort(items, kind='quicksort', ascending=True, na_position='last'):
 
     # specially handle Categorical
     if is_categorical_dtype(items):
-        return items.argsort(ascending=ascending, kind=kind)
+        if na_position not in ['first', 'last']:
+            raise ValueError('invalid na_position: {!r}'.format(na_position))
+        mask = isna(items)
+        cnt_null = mask.sum()
+        sorted_idx = items.argsort(ascending=ascending, kind=kind)
+        if ascending and na_position == 'last':
+            # NaN is coded as -1 and is listed in front after sorting
+            sorted_idx = np.roll(sorted_idx, -cnt_null)
+        elif not ascending and na_position == 'first':
+            # NaN is coded as -1 and is listed in the end after sorting
+            sorted_idx = np.roll(sorted_idx, cnt_null)
+        return sorted_idx
 
     items = np.asanyarray(items)
     idx = np.arange(len(items))

pandas/tests/frame/test_sorting.py

@@ -10,7 +10,7 @@
 from pandas.compat import lrange
 from pandas.api.types import CategoricalDtype
 from pandas import (DataFrame, Series, MultiIndex, Timestamp,
-                    date_range, NaT, IntervalIndex)
+                    date_range, NaT, IntervalIndex, Categorical)
 
 from pandas.util.testing import assert_series_equal, assert_frame_equal
 
@@ -598,3 +598,65 @@ def test_sort_index_intervalindex(self):
             closed='right')
         result = result.columns.levels[1].categories
         tm.assert_index_equal(result, expected)
+
+    def test_sort_index_na_position_with_categories(self):
+        # GH 22556
+        # Positioning missing value properly when column is Categorical.
+        df = pd.DataFrame({
+            'c': pd.Categorical(['A', np.nan, 'B', np.nan, 'C'],
+                                categories=['A', 'B', 'C'],
+                                ordered=True)})
+        result_ascending_na_first = df.sort_values(by='c',
+                                                   ascending=True,
+                                                   na_position='first')
+        expected_ascending_na_first = DataFrame({
+            'c': Categorical([np.nan, np.nan, 'A', 'B', 'C'],
+                             categories=['A', 'B', 'C'],
+                             ordered=True)}, index=[1, 3, 0, 2, 4])
+
+        assert_frame_equal(result_ascending_na_first,
+                           expected_ascending_na_first)
+
+        result_ascending_na_last = df.sort_values(by='c',
+                                                  ascending=True,
+                                                  na_position='last')
+        expected_ascending_na_last = DataFrame({
+            'c': Categorical(['A', 'B', 'C', np.nan, np.nan],
+                             categories=['A', 'B', 'C'],
+                             ordered=True)}, index=[0, 2, 4, 1, 3])
+
+        assert_frame_equal(result_ascending_na_last,
+                           expected_ascending_na_last)
+
+        result_descending_na_first = df.sort_values(by='c',
+                                                    ascending=False,
+                                                    na_position='first')
+        expected_descending_na_first = DataFrame({
+            'c': Categorical([np.nan, np.nan, 'C', 'B', 'A'],
+                             categories=['A', 'B', 'C'],
+                             ordered=True)}, index=[3, 1, 4, 2, 0])
+
+        assert_frame_equal(result_descending_na_first,
+                           expected_descending_na_first)
+
+        result_descending_na_last = df.sort_values(by='c',
+                                                   ascending=False,
+                                                   na_position='last')
+        expected_descending_na_last = DataFrame({
+            'c': Categorical(['C', 'B', 'A', np.nan, np.nan],
+                             categories=['A', 'B', 'C'],
+                             ordered=True)}, index=[4, 2, 0, 3, 1])
+
+        assert_frame_equal(result_descending_na_last,
+                           expected_descending_na_last)
+
+    def test_sort_index_na_position_with_categories_raises(self):
+        df = pd.DataFrame({
+            'c': pd.Categorical(['A', np.nan, 'B', np.nan, 'C'],
+                                categories=['A', 'B', 'C'],
+                                ordered=True)})
+
+        with pytest.raises(ValueError):
+            df.sort_values(by='c',
+                           ascending=False,
+                           na_position='bad_position')

pandas/tests/io/test_stata.py

@@ -997,7 +997,7 @@ def test_categorical_sorting(self, file):
         parsed = read_stata(getattr(self, file))
 
         # Sort based on codes, not strings
-        parsed = parsed.sort_values("srh")
+        parsed = parsed.sort_values("srh", na_position='first')
 
         # Don't sort index
         parsed.index = np.arange(parsed.shape[0])