Merge pull request #9479 from jreback/align

BUG: bug in partial setting of with a DatetimeIndex (GH9478)

Author: jreback

doc/source/whatsnew/v0.16.0.txt

@@ -186,7 +186,7 @@ Bug Fixes
 - Bug in Panel indexing with an object-like (:issue:`9140`)
 - Bug in the returned ``Series.dt.components`` index was reset to the default index (:issue:`9247`)
 - Bug in ``Categorical.__getitem__/__setitem__`` with listlike input getting incorrect results from indexer coercion (:issue:`9469`)
-
+- Bug in partial setting with a DatetimeIndex (:issue:`9478`)
 - Fixed bug in ``to_sql`` when mapping a ``Timestamp`` object column (datetime
   column with timezone info) to the according sqlalchemy type (:issue:`9085`).
 - Fixed bug in ``to_sql`` ``dtype`` argument not accepting an instantiated

pandas/core/common.py

@@ -1011,6 +1011,27 @@ def conv(r, dtype):
     return [conv(r, dtype) for r, dtype in zip(result, dtypes)]
 
 
+def _infer_fill_value(val):
+    """
+    infer the fill value for the nan/NaT from the provided scalar/ndarray/list-like
+    if we are a NaT, return the correct dtyped element to provide proper block construction
+
+    """
+
+    if not is_list_like(val):
+        val = [val]
+    val = np.array(val,copy=False)
+    if is_datetimelike(val):
+        return np.array('NaT',dtype=val.dtype)
+    elif is_object_dtype(val.dtype):
+        dtype = lib.infer_dtype(_ensure_object(val))
+        if dtype in ['datetime','datetime64']:
+            return np.array('NaT',dtype=_NS_DTYPE)
+        elif dtype in ['timedelta','timedelta64']:
+            return np.array('NaT',dtype=_TD_DTYPE)
+    return np.nan
+
+
 def _infer_dtype_from_scalar(val):
     """ interpret the dtype from a scalar, upcast floats and ints
         return the new value and the dtype """

pandas/core/index.py

@@ -345,7 +345,10 @@ def _get_attributes_dict(self):
         return dict([ (k,getattr(self,k,None)) for k in self._attributes])
 
     def view(self, cls=None):
-        if cls is not None and not issubclass(cls, Index):
+
+        # we need to see if we are subclassing an
+        # index type here
+        if cls is not None and not hasattr(cls,'_typ'):
             result = self._data.view(cls)
         else:
             result = self._shallow_copy()

pandas/core/indexing.py

@@ -8,7 +8,7 @@
 from pandas.core.common import (_is_bool_indexer, is_integer_dtype,
                                 _asarray_tuplesafe, is_list_like, isnull,
                                 ABCSeries, ABCDataFrame, ABCPanel, is_float,
-                                _values_from_object)
+                                _values_from_object, _infer_fill_value)
 import pandas.lib as lib
 
 import numpy as np
@@ -238,7 +238,9 @@ def _setitem_with_indexer(self, indexer, value):
                             self.obj[key] = value
                             return self.obj
 
-                        self.obj[key] = np.nan
+
+                        # add a new item with the dtype setup
+                        self.obj[key] = _infer_fill_value(value)
 
                         new_indexer = _convert_from_missing_indexer_tuple(
                             indexer, self.obj.axes)

pandas/tests/test_index.py

@@ -297,6 +297,9 @@ def test_view(self):
         i_view = i.view()
         self.assertEqual(i_view.name, 'Foo')
 
+        # with arguments
+        self.assertRaises(TypeError, lambda : i.view('i8'))
+
     def test_legacy_pickle_identity(self):
 
         # GH 8431
@@ -1469,6 +1472,12 @@ def test_view(self):
         i_view = i.view()
         self.assertEqual(i_view.name, 'Foo')
 
+        i_view = i.view('i8')
+        tm.assert_index_equal(i, Int64Index(i_view))
+
+        i_view = i.view(Int64Index)
+        tm.assert_index_equal(i, Int64Index(i_view))
+
     def test_coerce_list(self):
         # coerce things
         arr = Index([1, 2, 3, 4])
@@ -1856,7 +1865,21 @@ def test_slice_keep_name(self):
         idx = Int64Index([1, 2], name='asdf')
         self.assertEqual(idx.name, idx[1:].name)
 
-class TestDatetimeIndex(Base, tm.TestCase):
+class DatetimeLike(Base):
+
+    def test_view(self):
+
+        i = self.create_index()
+
+        i_view = i.view('i8')
+        result = self._holder(i)
+        tm.assert_index_equal(result, i)
+
+        i_view = i.view(self._holder)
+        result = self._holder(i)
+        tm.assert_index_equal(result, i)
+
+class TestDatetimeIndex(DatetimeLike, tm.TestCase):
     _holder = DatetimeIndex
     _multiprocess_can_split_ = True
 
@@ -1926,7 +1949,7 @@ def test_time_overflow_for_32bit_machines(self):
         self.assertEqual(len(idx2), periods)
 
 
-class TestPeriodIndex(Base, tm.TestCase):
+class TestPeriodIndex(DatetimeLike, tm.TestCase):
     _holder = PeriodIndex
     _multiprocess_can_split_ = True
 
@@ -1936,7 +1959,7 @@ def create_index(self):
     def test_pickle_compat_construction(self):
         pass
 
-class TestTimedeltaIndex(Base, tm.TestCase):
+class TestTimedeltaIndex(DatetimeLike, tm.TestCase):
     _holder = TimedeltaIndex
     _multiprocess_can_split_ = True
 

pandas/tests/test_indexing.py

@@ -3158,6 +3158,19 @@ def f():
         df.loc[3] = [6,7]
         assert_frame_equal(df,DataFrame([[6,7]],index=[3],columns=['A','B'],dtype='float64'))
 
+    def test_partial_setting_with_datetimelike_dtype(self):
+
+        # GH9478
+        # a datetimeindex alignment issue with partial setting
+        df = pd.DataFrame(np.arange(6.).reshape(3,2), columns=list('AB'),
+                          index=pd.date_range('1/1/2000', periods=3, freq='1H'))
+        expected = df.copy()
+        expected['C'] = [expected.index[0]] + [pd.NaT,pd.NaT]
+
+        mask = df.A < 1
+        df.loc[mask, 'C'] = df.loc[mask].index
+        assert_frame_equal(df, expected)
+
     def test_series_partial_set(self):
         # partial set with new index
         # Regression from GH4825