Skip to content

BUG: Don't cast categorical nan to int #28438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Sep 18, 2019
1 change: 1 addition & 0 deletions doc/source/whatsnew/v1.0.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ Categorical
^^^^^^^^^^^

- Added test to assert the :func:`fillna` raises the correct ValueError message when the value isn't a value from categories (:issue:`13628`)
- Bug in :meth:`Categorical.astype` where ``NaN`` values were handled incorrectly when casting to int (:issue:`28406`)
-
-

Expand Down
3 changes: 3 additions & 0 deletions pandas/core/arrays/categorical.py
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,9 @@ def astype(self, dtype: Dtype, copy: bool = True) -> ArrayLike:
if dtype == self.dtype:
return self
return self._set_dtype(dtype)
if is_integer_dtype(dtype) and self.isna().any():
msg = "Cannot cast to int."
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ValueError: cannot convert float NaN to integer is what we do now on
pd.Series([1,2,np.nan],dtype='Int64').astype('int') so would replicate this message

raise ValueError(msg)
return np.array(self, dtype=dtype, copy=copy)

@cache_readonly
Expand Down
8 changes: 4 additions & 4 deletions pandas/core/indexes/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -4713,13 +4713,13 @@ def set_value(self, arr, key, value):
@Appender(_index_shared_docs["get_indexer_non_unique"] % _index_doc_kwargs)
def get_indexer_non_unique(self, target):
target = ensure_index(target)
if is_categorical(target):
target = target.astype(target.dtype.categories.dtype)
pself, ptarget = self._maybe_promote(target)
if pself is not self or ptarget is not target:
return pself.get_indexer_non_unique(ptarget)

if self.is_all_dates:
if is_categorical(target):
tgt_values = np.asarray(target)
elif self.is_all_dates:
tgt_values = target.asi8
else:
tgt_values = target._ndarray_values
Expand All @@ -4731,7 +4731,7 @@ def get_indexer_for(self, target, **kwargs):
"""
Guaranteed return of an indexer even when non-unique.

This dispatches to get_indexer or get_indexer_nonunique
This dispatches to get_indexer or get_indexer_non_unique
as appropriate.

Returns
Expand Down
10 changes: 8 additions & 2 deletions pandas/tests/extension/test_categorical.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import pytest

import pandas as pd
from pandas import Categorical
from pandas import Categorical, CategoricalIndex
from pandas.api.types import CategoricalDtype
from pandas.tests.extension import base
import pandas.util.testing as tm
Expand Down Expand Up @@ -197,7 +197,13 @@ def test_searchsorted(self, data_for_sorting):


class TestCasting(base.BaseCastingTests):
pass
@pytest.mark.parametrize("cls", [Categorical, CategoricalIndex])
def test_cast_nan_to_int(self, cls):
# GH 28406
s = cls([0, 1, np.nan])

with pytest.raises((ValueError, TypeError), match="Cannot cast"):
s.astype(int)


class TestArithmeticOps(base.BaseArithmeticOpsTests):
Expand Down