REF (feedback): move maybe_box_datetimelike common.py -> dtypes/cast.py

Author: arw2019

doc/source/whatsnew/v1.2.0.rst

@@ -456,6 +456,7 @@ Indexing
 - Bug in :meth:`Series.loc.__getitem__` with a non-unique :class:`MultiIndex` and an empty-list indexer (:issue:`13691`)
 - Bug in indexing on a :class:`Series` or :class:`DataFrame` with a :class:`MultiIndex` with a level named "0" (:issue:`37194`)
 - Bug in :meth:`Series.__getitem__` when using an unsigned integer array as an indexer giving incorrect results or segfaulting instead of raising ``KeyError`` (:issue:`37218`)
+- Bug in :meth:`Index.where` incorrectly casting numeric values to strings (:issue:`37591`)
 
 Missing
 ^^^^^^^

pandas/conftest.py

@@ -290,6 +290,16 @@ def unique_nulls_fixture(request):
 # ----------------------------------------------------------------
 # Classes
 # ----------------------------------------------------------------
+
+
+@pytest.fixture(params=[pd.DataFrame, pd.Series])
+def frame_or_series(request):
+    """
+    Fixture to parametrize over DataFrame and Series.
+    """
+    return request.param
+
+
 @pytest.fixture(
     params=[pd.Index, pd.Series], ids=["index", "series"]  # type: ignore[list-item]
 )

pandas/core/computation/align.py

@@ -38,8 +38,7 @@ def _align_core_single_unary_op(
 def _zip_axes_from_type(
     typ: Type[FrameOrSeries], new_axes: Sequence[int]
 ) -> Dict[str, int]:
-    axes = {name: new_axes[i] for i, name in enumerate(typ._AXIS_ORDERS)}
-    return axes
+    return {name: new_axes[i] for i, name in enumerate(typ._AXIS_ORDERS)}
 
 
 def _any_pandas_objects(terms) -> bool:
@@ -186,8 +185,11 @@ def reconstruct_object(typ, obj, axes, dtype):
         # The condition is to distinguish 0-dim array (returned in case of
         # scalar) and 1 element array
         # e.g. np.array(0) and np.array([0])
-        if len(obj.shape) == 1 and len(obj) == 1:
-            if not isinstance(ret_value, np.ndarray):
-                ret_value = np.array([ret_value]).astype(res_t)
+        if (
+            len(obj.shape) == 1
+            and len(obj) == 1
+            and not isinstance(ret_value, np.ndarray)
+        ):
+            ret_value = np.array([ret_value]).astype(res_t)
 
     return ret_value

pandas/core/computation/eval.py

@@ -52,12 +52,11 @@ def _check_engine(engine: Optional[str]) -> str:
     # TODO: validate this in a more general way (thinking of future engines
     # that won't necessarily be import-able)
     # Could potentially be done on engine instantiation
-    if engine == "numexpr":
-        if not NUMEXPR_INSTALLED:
-            raise ImportError(
-                "'numexpr' is not installed or an unsupported version. Cannot use "
-                "engine='numexpr' for query/eval if 'numexpr' is not installed"
-            )
+    if engine == "numexpr" and not NUMEXPR_INSTALLED:
+        raise ImportError(
+            "'numexpr' is not installed or an unsupported version. Cannot use "
+            "engine='numexpr' for query/eval if 'numexpr' is not installed"
+        )
 
     return engine
 

pandas/core/computation/expr.py

@@ -496,15 +496,14 @@ def _maybe_evaluate_binop(
                 f"'{lhs.type}' and '{rhs.type}'"
             )
 
-        if self.engine != "pytables":
-            if (
-                res.op in CMP_OPS_SYMS
-                and getattr(lhs, "is_datetime", False)
-                or getattr(rhs, "is_datetime", False)
-            ):
-                # all date ops must be done in python bc numexpr doesn't work
-                # well with NaT
-                return self._maybe_eval(res, self.binary_ops)
+        if self.engine != "pytables" and (
+            res.op in CMP_OPS_SYMS
+            and getattr(lhs, "is_datetime", False)
+            or getattr(rhs, "is_datetime", False)
+        ):
+            # all date ops must be done in python bc numexpr doesn't work
+            # well with NaT
+            return self._maybe_eval(res, self.binary_ops)
 
         if res.op in eval_in_python:
             # "in"/"not in" ops are always evaluated in python

pandas/core/computation/pytables.py

@@ -378,14 +378,14 @@ def prune(self, klass):
         operand = self.operand
         operand = operand.prune(klass)
 
-        if operand is not None:
-            if issubclass(klass, ConditionBinOp):
-                if operand.condition is not None:
-                    return operand.invert()
-            elif issubclass(klass, FilterBinOp):
-                if operand.filter is not None:
-                    return operand.invert()
-
+        if operand is not None and (
+            issubclass(klass, ConditionBinOp)
+            and operand.condition is not None
+            or not issubclass(klass, ConditionBinOp)
+            and issubclass(klass, FilterBinOp)
+            and operand.filter is not None
+        ):
+            return operand.invert()
         return None
 
 

pandas/core/computation/scope.py

@@ -144,8 +144,7 @@ def __init__(
     def __repr__(self) -> str:
         scope_keys = _get_pretty_string(list(self.scope.keys()))
         res_keys = _get_pretty_string(list(self.resolvers.keys()))
-        unicode_str = f"{type(self).__name__}(scope={scope_keys}, resolvers={res_keys})"
-        return unicode_str
+        return f"{type(self).__name__}(scope={scope_keys}, resolvers={res_keys})"
 
     @property
     def has_resolvers(self) -> bool:

pandas/core/frame.py

@@ -132,7 +132,13 @@
 from pandas.core.construction import extract_array
 from pandas.core.generic import NDFrame, _shared_docs
 from pandas.core.indexes import base as ibase
-from pandas.core.indexes.api import Index, ensure_index, ensure_index_from_sequences
+from pandas.core.indexes.api import (
+    DatetimeIndex,
+    Index,
+    PeriodIndex,
+    ensure_index,
+    ensure_index_from_sequences,
+)
 from pandas.core.indexes.multi import MultiIndex, maybe_droplevels
 from pandas.core.indexing import check_bool_indexer, convert_to_index_sliceable
 from pandas.core.internals import BlockManager
@@ -9254,6 +9260,9 @@ def to_timestamp(
 
         axis_name = self._get_axis_name(axis)
         old_ax = getattr(self, axis_name)
+        if not isinstance(old_ax, PeriodIndex):
+            raise TypeError(f"unsupported Type {type(old_ax).__name__}")
+
         new_ax = old_ax.to_timestamp(freq=freq, how=how)
 
         setattr(new_obj, axis_name, new_ax)
@@ -9283,6 +9292,9 @@ def to_period(self, freq=None, axis: Axis = 0, copy: bool = True) -> DataFrame:
 
         axis_name = self._get_axis_name(axis)
         old_ax = getattr(self, axis_name)
+        if not isinstance(old_ax, DatetimeIndex):
+            raise TypeError(f"unsupported Type {type(old_ax).__name__}")
+
         new_ax = old_ax.to_period(freq=freq)
 
         setattr(new_obj, axis_name, new_ax)

pandas/core/indexes/base.py

@@ -40,7 +40,6 @@
     ensure_int64,
     ensure_object,
     ensure_platform_int,
-    is_bool,
     is_bool_dtype,
     is_categorical_dtype,
     is_datetime64_any_dtype,
@@ -4079,23 +4078,16 @@ def where(self, cond, other=None):
         if other is None:
             other = self._na_value
 
-        dtype = self.dtype
         values = self.values
 
-        if is_bool(other) or is_bool_dtype(other):
-
-            # bools force casting
-            values = values.astype(object)
-            dtype = None
+        try:
+            self._validate_fill_value(other)
+        except (ValueError, TypeError):
+            return self.astype(object).where(cond, other)
 
         values = np.where(cond, values, other)
 
-        if self._is_numeric_dtype and np.any(isna(values)):
-            # We can't coerce to the numeric dtype of "self" (unless
-            # it's float) if there are NaN values in our output.
-            dtype = None
-
-        return Index(values, dtype=dtype, name=self.name)
+        return Index(values, name=self.name)
 
     # construction helpers
     @final

pandas/core/indexes/datetimelike.py

@@ -482,16 +482,9 @@ def isin(self, values, level=None):
 
     @Appender(Index.where.__doc__)
     def where(self, cond, other=None):
-        values = self._data._ndarray
+        other = self._data._validate_setitem_value(other)
 
-        try:
-            other = self._data._validate_setitem_value(other)
-        except (TypeError, ValueError) as err:
-            # Includes tzawareness mismatch and IncompatibleFrequencyError
-            oth = getattr(other, "dtype", other)
-            raise TypeError(f"Where requires matching dtype, not {oth}") from err
-
-        result = np.where(cond, values, other)
+        result = np.where(cond, self._data._ndarray, other)
         arr = self._data._from_backing_data(result)
         return type(self)._simple_new(arr, name=self.name)
 

pandas/core/indexes/numeric.py

@@ -121,6 +121,8 @@ def _validate_fill_value(self, value):
             # force conversion to object
             # so we don't lose the bools
             raise TypeError
+        if isinstance(value, str):
+            raise TypeError
 
         return value
 

pandas/tests/frame/methods/test_at_time.py

@@ -4,38 +4,57 @@
 import pytest
 import pytz
 
+from pandas._libs.tslibs import timezones
+
 from pandas import DataFrame, date_range
 import pandas._testing as tm
 
 
 class TestAtTime:
-    def test_at_time(self):
+    @pytest.mark.parametrize("tzstr", ["US/Eastern", "dateutil/US/Eastern"])
+    def test_localized_at_time(self, tzstr, frame_or_series):
+        tz = timezones.maybe_get_tz(tzstr)
+
+        rng = date_range("4/16/2012", "5/1/2012", freq="H")
+        ts = frame_or_series(np.random.randn(len(rng)), index=rng)
+
+        ts_local = ts.tz_localize(tzstr)
+
+        result = ts_local.at_time(time(10, 0))
+        expected = ts.at_time(time(10, 0)).tz_localize(tzstr)
+        tm.assert_equal(result, expected)
+        assert timezones.tz_compare(result.index.tz, tz)
+
+    def test_at_time(self, frame_or_series):
         rng = date_range("1/1/2000", "1/5/2000", freq="5min")
         ts = DataFrame(np.random.randn(len(rng), 2), index=rng)
+        if frame_or_series is not DataFrame:
+            ts = ts[0]
         rs = ts.at_time(rng[1])
         assert (rs.index.hour == rng[1].hour).all()
         assert (rs.index.minute == rng[1].minute).all()
         assert (rs.index.second == rng[1].second).all()
 
         result = ts.at_time("9:30")
         expected = ts.at_time(time(9, 30))
-        tm.assert_frame_equal(result, expected)
-
-        result = ts.loc[time(9, 30)]
-        expected = ts.loc[(rng.hour == 9) & (rng.minute == 30)]
-
-        tm.assert_frame_equal(result, expected)
+        tm.assert_equal(result, expected)
 
+    def test_at_time_midnight(self, frame_or_series):
         # midnight, everything
         rng = date_range("1/1/2000", "1/31/2000")
         ts = DataFrame(np.random.randn(len(rng), 3), index=rng)
+        if frame_or_series is not DataFrame:
+            ts = ts[0]
 
         result = ts.at_time(time(0, 0))
-        tm.assert_frame_equal(result, ts)
+        tm.assert_equal(result, ts)
 
+    def test_at_time_nonexistent(self, frame_or_series):
         # time doesn't exist
         rng = date_range("1/1/2012", freq="23Min", periods=384)
-        ts = DataFrame(np.random.randn(len(rng), 2), rng)
+        ts = DataFrame(np.random.randn(len(rng)), rng)
+        if frame_or_series is not DataFrame:
+            ts = ts[0]
         rs = ts.at_time("16:00")
         assert len(rs) == 0
 
@@ -62,12 +81,14 @@ def test_at_time_tz(self):
         expected = df.iloc[1:2]
         tm.assert_frame_equal(result, expected)
 
-    def test_at_time_raises(self):
+    def test_at_time_raises(self, frame_or_series):
         # GH#20725
-        df = DataFrame([[1, 2, 3], [4, 5, 6]])
+        obj = DataFrame([[1, 2, 3], [4, 5, 6]])
+        if frame_or_series is not DataFrame:
+            obj = obj[0]
         msg = "Index must be DatetimeIndex"
         with pytest.raises(TypeError, match=msg):  # index is not a DatetimeIndex
-            df.at_time("00:00")
+            obj.at_time("00:00")
 
     @pytest.mark.parametrize("axis", ["index", "columns", 0, 1])
     def test_at_time_axis(self, axis):