BUG: Series(mixed_tz_datetime_objs, dtype=dt64tz)

doc/source/whatsnew/v2.0.0.rst

@@ -350,7 +350,7 @@ Timedelta
 
 Timezones
 ^^^^^^^^^
--
+- Bug in :meth:`Series.astype` and :meth:`DataFrame.astype` with object-dtype containing multiple timezone-aware ``datetime`` objects with heterogeneous timezones to a :class:`DatetimeTZDtype` incorrectly raising (:issue:`32581`)
 -
 
 Numeric

pandas/_libs/tslib.pyi

@@ -28,3 +28,7 @@ def array_to_datetime(
 ) -> tuple[np.ndarray, tzinfo | None]: ...
 
 # returned ndarray may be object dtype or datetime64[ns]
+
+def array_to_datetime_with_tz(
+    values: npt.NDArray[np.object_], tz: tzinfo
+) -> npt.NDArray[np.int64]: ...

pandas/_libs/tslib.pyx

@@ -6,6 +6,7 @@ from cpython.datetime cimport (
     import_datetime,
     tzinfo,
 )
+from cpython.object cimport PyObject
 
 # import datetime C API
 import_datetime()
@@ -862,3 +863,50 @@ cdef inline bint _parse_today_now(str val, int64_t* iresult, bint utc):
         iresult[0] = Timestamp.today().value
         return True
     return False
+
+
+def array_to_datetime_with_tz(ndarray values, tzinfo tz):
+    """
+    Vectorized analogue to pd.Timestamp(value, tz=tz)
+
+    values has object-dtype, unrestricted ndim.
+
+    Major differences between this and array_to_datetime with utc=True
+        - np.datetime64 objects are treated as _wall_ times.
+        - tznaive datetimes are treated as _wall_ times.
+    """
+    cdef:
+        ndarray result = cnp.PyArray_EMPTY(values.ndim, values.shape, cnp.NPY_INT64, 0)
+        cnp.broadcast mi = cnp.PyArray_MultiIterNew2(result, values)
+        Py_ssize_t i, n = values.size
+        object item
+        int64_t ival
+        datetime ts
+
+    for i in range(n):
+        # Analogous to `item = values[i]`
+        item = <object>(<PyObject**>cnp.PyArray_MultiIter_DATA(mi, 1))[0]
+
+        if checknull_with_nat_and_na(item):
+            # this catches pd.NA which would raise in the Timestamp constructor
+            ival = NPY_NAT
+
+        else:
+            ts = Timestamp(item)
+            if ts is NaT:
+                ival = NPY_NAT
+            else:
+                if ts.tz is not None:
+                    ts = ts.tz_convert(tz)
+                else:
+                    # datetime64, tznaive pydatetime, int, float
+                    ts = ts.tz_localize(tz)
+                ts = ts._as_unit("ns")
+                ival = ts.value
+
+        # Analogous to: result[i] = ival
+        (<int64_t*>cnp.PyArray_MultiIter_DATA(mi, 0))[0] = ival
+
+        cnp.PyArray_MultiIter_NEXT(mi)
+
+    return result

pandas/core/arrays/datetimes.py

@@ -2029,6 +2029,11 @@ def _sequence_to_dt64ns(
         copy = False
         if lib.infer_dtype(data, skipna=False) == "integer":
             data = data.astype(np.int64)
+        elif tz is not None and ambiguous == "raise":
+            # TODO: yearfirst/dayfirst/etc?
+            obj_data = np.asarray(data, dtype=object)
+            i8data = tslib.array_to_datetime_with_tz(obj_data, tz)
+            return i8data.view(DT64NS_DTYPE), tz, None
         else:
             # data comes back here as either i8 to denote UTC timestamps
             #  or M8[ns] to denote wall times

pandas/tests/arrays/test_datetimelike.py

@@ -292,19 +292,7 @@ def test_searchsorted(self):
         assert result == 10
 
     @pytest.mark.parametrize("box", [None, "index", "series"])
-    def test_searchsorted_castable_strings(self, arr1d, box, request, string_storage):
-        if isinstance(arr1d, DatetimeArray):
-            tz = arr1d.tz
-            ts1, ts2 = arr1d[1:3]
-            if tz is not None and ts1.tz.tzname(ts1) != ts2.tz.tzname(ts2):
-                # If we have e.g. tzutc(), when we cast to string and parse
-                #  back we get pytz.UTC, and then consider them different timezones
-                #  so incorrectly raise.
-                mark = pytest.mark.xfail(
-                    raises=TypeError, reason="timezone comparisons inconsistent"
-                )
-                request.node.add_marker(mark)
-
+    def test_searchsorted_castable_strings(self, arr1d, box, string_storage):
         arr = arr1d
         if box is None:
             pass
@@ -461,19 +449,8 @@ def test_setitem_object_dtype(self, box, arr1d):
 
         tm.assert_equal(arr1d, expected)
 
-    def test_setitem_strs(self, arr1d, request):
+    def test_setitem_strs(self, arr1d):
         # Check that we parse strs in both scalar and listlike
-        if isinstance(arr1d, DatetimeArray):
-            tz = arr1d.tz
-            ts1, ts2 = arr1d[-2:]
-            if tz is not None and ts1.tz.tzname(ts1) != ts2.tz.tzname(ts2):
-                # If we have e.g. tzutc(), when we cast to string and parse
-                #  back we get pytz.UTC, and then consider them different timezones
-                #  so incorrectly raise.
-                mark = pytest.mark.xfail(
-                    raises=TypeError, reason="timezone comparisons inconsistent"
-                )
-                request.node.add_marker(mark)
 
         # Setting list-like of strs
         expected = arr1d.copy()

pandas/tests/indexes/datetimes/test_constructors.py

@@ -466,25 +466,57 @@ def test_construction_dti_with_mixed_timezones(self):
                 name="idx",
             )
 
-        with pytest.raises(ValueError, match=msg):
-            DatetimeIndex(
-                [
-                    Timestamp("2011-01-01 10:00"),
-                    Timestamp("2011-01-02 10:00", tz="US/Eastern"),
-                ],
-                tz="Asia/Tokyo",
-                name="idx",
-            )
+        # pre-2.0 this raised bc of awareness mismatch. in 2.0 with a tz#
+        #  specified we behave as if this was called pointwise, so
+        #  the naive Timestamp is treated as a wall time.
+        dti = DatetimeIndex(
+            [
+                Timestamp("2011-01-01 10:00"),
+                Timestamp("2011-01-02 10:00", tz="US/Eastern"),
+            ],
+            tz="Asia/Tokyo",
+            name="idx",
+        )
+        expected = DatetimeIndex(
+            [
+                Timestamp("2011-01-01 10:00", tz="Asia/Tokyo"),
+                Timestamp("2011-01-02 10:00", tz="US/Eastern").tz_convert("Asia/Tokyo"),
+            ],
+            tz="Asia/Tokyo",
+            name="idx",
+        )
+        tm.assert_index_equal(dti, expected)
 
-        with pytest.raises(ValueError, match=msg):
-            DatetimeIndex(
-                [
-                    Timestamp("2011-01-01 10:00", tz="Asia/Tokyo"),
-                    Timestamp("2011-01-02 10:00", tz="US/Eastern"),
-                ],
-                tz="US/Eastern",
-                name="idx",
-            )
+        # pre-2.0 mixed-tz scalars raised even if a tz/dtype was specified.
+        #  as of 2.0 we successfully return the requested tz/dtype
+        dti = DatetimeIndex(
+            [
+                Timestamp("2011-01-01 10:00", tz="Asia/Tokyo"),
+                Timestamp("2011-01-02 10:00", tz="US/Eastern"),
+            ],
+            tz="US/Eastern",
+            name="idx",
+        )
+        expected = DatetimeIndex(
+            [
+                Timestamp("2011-01-01 10:00", tz="Asia/Tokyo").tz_convert("US/Eastern"),
+                Timestamp("2011-01-02 10:00", tz="US/Eastern"),
+            ],
+            tz="US/Eastern",
+            name="idx",
+        )
+        tm.assert_index_equal(dti, expected)
+
+        # same thing but pass dtype instead of tz
+        dti = DatetimeIndex(
+            [
+                Timestamp("2011-01-01 10:00", tz="Asia/Tokyo"),
+                Timestamp("2011-01-02 10:00", tz="US/Eastern"),
+            ],
+            dtype="M8[ns, US/Eastern]",
+            name="idx",
+        )
+        tm.assert_index_equal(dti, expected)
 
     def test_construction_base_constructor(self):
         arr = [Timestamp("2011-01-01"), pd.NaT, Timestamp("2011-01-03")]

pandas/tests/series/methods/test_astype.py

@@ -97,6 +97,20 @@ def test_astype_dict_like(self, dtype_class):
 
 
 class TestAstype:
+    def test_astype_mixed_object_to_dt64tz(self):
+        # pre-2.0 this raised ValueError bc of tz mismatch
+        # xref GH#32581
+        ts = Timestamp("2016-01-04 05:06:07", tz="US/Pacific")
+        ts2 = ts.tz_convert("Asia/Tokyo")
+
+        ser = Series([ts, ts2], dtype=object)
+        res = ser.astype("datetime64[ns, Europe/Brussels]")
+        expected = Series(
+            [ts.tz_convert("Europe/Brussels"), ts2.tz_convert("Europe/Brussels")],
+            dtype="datetime64[ns, Europe/Brussels]",
+        )
+        tm.assert_series_equal(res, expected)
+
     @pytest.mark.parametrize("dtype", np.typecodes["All"])
     def test_astype_empty_constructor_equality(self, dtype):
         # see GH#15524