Skip to content

REF: remove need to override get_indexer_non_unique in DatetimeIndexOpsMixin #33792

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 25, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pandas/_libs/index.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,10 @@ cdef class DatetimeEngine(Int64Engine):
except KeyError:
raise KeyError(val)

def get_indexer_non_unique(self, targets):
# we may get datetime64[ns] or timedelta64[ns], cast these to int64
return super().get_indexer_non_unique(targets.view("i8"))

def get_indexer(self, values):
self._ensure_mapping_populated()
if values.dtype != self._get_box_dtype():
Expand Down
27 changes: 24 additions & 3 deletions pandas/core/indexes/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from pandas._libs.tslibs import OutOfBoundsDatetime, Timestamp
from pandas._libs.tslibs.period import IncompatibleFrequency
from pandas._libs.tslibs.timezones import tz_compare
from pandas._typing import Label
from pandas._typing import DtypeObj, Label
from pandas.compat import set_function_name
from pandas.compat.numpy import function as nv
from pandas.util._decorators import Appender, Substitution, cache_readonly, doc
Expand Down Expand Up @@ -4626,6 +4626,10 @@ def get_indexer_non_unique(self, target):
if pself is not self or ptarget is not target:
return pself.get_indexer_non_unique(ptarget)

if not self._is_comparable_dtype(target.dtype):
no_matches = -1 * np.ones(self.shape, dtype=np.intp)
return no_matches, no_matches

if is_categorical_dtype(target.dtype):
tgt_values = np.asarray(target)
else:
Expand All @@ -4651,16 +4655,33 @@ def get_indexer_for(self, target, **kwargs):
indexer, _ = self.get_indexer_non_unique(target, **kwargs)
return indexer

def _maybe_promote(self, other):
# A hack, but it works
def _maybe_promote(self, other: "Index"):
"""
When dealing with an object-dtype Index and a non-object Index, see
if we can upcast the object-dtype one to improve performance.
"""

if self.inferred_type == "date" and isinstance(other, ABCDatetimeIndex):
return type(other)(self), other
elif self.inferred_type == "timedelta" and isinstance(other, ABCTimedeltaIndex):
# TODO: we dont have tests that get here
return type(other)(self), other
elif self.inferred_type == "boolean":
if not is_object_dtype(self.dtype):
return self.astype("object"), other.astype("object")

if not is_object_dtype(self.dtype) and is_object_dtype(other.dtype):
# Reverse op so we dont need to re-implement on the subclasses
other, self = other._maybe_promote(self)

return self, other

def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
"""
Can we compare values of the given dtype to our own?
"""
return True

def groupby(self, values) -> PrettyDict[Hashable, np.ndarray]:
"""
Group the index labels by a given array of values.
Expand Down
26 changes: 2 additions & 24 deletions pandas/core/indexes/datetimelike.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,13 @@

from pandas._libs import NaT, iNaT, join as libjoin, lib
from pandas._libs.tslibs import timezones
from pandas._typing import DtypeObj, Label
from pandas._typing import Label
from pandas.compat.numpy import function as nv
from pandas.errors import AbstractMethodError
from pandas.util._decorators import Appender, cache_readonly, doc

from pandas.core.dtypes.common import (
ensure_int64,
ensure_platform_int,
is_bool_dtype,
is_dtype_equal,
is_integer,
Expand All @@ -31,7 +30,7 @@
from pandas.core.arrays.datetimelike import DatetimeLikeArrayMixin
from pandas.core.base import IndexOpsMixin
import pandas.core.indexes.base as ibase
from pandas.core.indexes.base import Index, _index_shared_docs, ensure_index
from pandas.core.indexes.base import Index, _index_shared_docs
from pandas.core.indexes.extension import (
ExtensionIndex,
inherit_names,
Expand Down Expand Up @@ -99,12 +98,6 @@ class DatetimeIndexOpsMixin(ExtensionIndex):
def is_all_dates(self) -> bool:
return True

def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
"""
Can we compare values of the given dtype to our own?
"""
raise AbstractMethodError(self)

# ------------------------------------------------------------------------
# Abstract data attributes

Expand Down Expand Up @@ -430,21 +423,6 @@ def _partial_date_slice(
# try to find the dates
return (lhs_mask & rhs_mask).nonzero()[0]

@Appender(Index.get_indexer_non_unique.__doc__)
def get_indexer_non_unique(self, target):
target = ensure_index(target)
pself, ptarget = self._maybe_promote(target)
if pself is not self or ptarget is not target:
return pself.get_indexer_non_unique(ptarget)

if not self._is_comparable_dtype(target.dtype):
no_matches = -1 * np.ones(self.shape, dtype=np.intp)
return no_matches, no_matches

tgt_values = target.asi8
indexer, missing = self._engine.get_indexer_non_unique(tgt_values)
return ensure_platform_int(indexer), missing

# --------------------------------------------------------------------

__add__ = make_wrapped_arith_op("__add__")
Expand Down
5 changes: 0 additions & 5 deletions pandas/core/indexes/datetimes.py
Original file line number Diff line number Diff line change
Expand Up @@ -538,11 +538,6 @@ def _validate_partial_date_slice(self, reso: str):
# _parsed_string_to_bounds allows it.
raise KeyError

def _maybe_promote(self, other):
if other.inferred_type == "date":
other = DatetimeIndex(other)
return self, other

def get_loc(self, key, method=None, tolerance=None):
"""
Get integer location for requested label
Expand Down
5 changes: 0 additions & 5 deletions pandas/core/indexes/timedeltas.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,11 +197,6 @@ def astype(self, dtype, copy=True):
return Index(result.astype("i8"), name=self.name)
return DatetimeIndexOpsMixin.astype(self, dtype, copy=copy)

def _maybe_promote(self, other):
if other.inferred_type == "timedelta":
other = TimedeltaIndex(other)
return self, other

def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
"""
Can we compare values of the given dtype to our own?
Expand Down