BUG: not setting placement on reindex_with_indexers

ENH: extend index.reindex to handle non_unique indicies (rather than raising)

TST: more tests/optimizations for dup_columns

Author: jreback

RELEASE.rst

@@ -136,6 +136,7 @@ pandas 0.11.1
     - Concat to produce a non-unique columns when duplicates are across dtypes is fixed (GH3602_)
     - Non-unique indexing with a slice via ``loc`` and friends fixed (GH3659_)
     - Allow insert/delete to non-unique columns (GH3679_)
+    - Extend ``reindex`` to correctly deal with non-unique indices (GH3679_)
   - Fixed bug in groupby with empty series referencing a variable before assignment. (GH3510_)
   - Fixed bug in mixed-frame assignment with aligned series (GH3492_)
   - Fixed bug in selecting month/quarter/year from a series would not select the time element

pandas/core/index.py

@@ -940,8 +940,15 @@ def reindex(self, target, method=None, level=None, limit=None):
             if self.equals(target):
                 indexer = None
             else:
-                indexer = self.get_indexer(target, method=method,
-                                           limit=limit)
+                if self.is_unique:
+                    indexer = self.get_indexer(target, method=method,
+                                               limit=limit)
+                else:
+                    if method is not None or limit is not None:
+                        raise ValueError("cannot reindex a non-unique index "
+                                         "with a method or limit")
+                    indexer, missing = self.get_indexer_non_unique(target)
+
         return target, indexer
 
     def join(self, other, how='left', level=None, return_indexers=False):

pandas/core/indexing.py

@@ -457,7 +457,7 @@ def _reindex(keys, level=None):
             else:
                 level = None
 
-            if labels.is_unique:
+            if labels.is_unique and Index(keyarr).is_unique:
                 return _reindex(keyarr, level=level)
             else:
                 indexer, missing = labels.get_indexer_non_unique(keyarr)
@@ -991,7 +991,6 @@ def _slice(self, indexer, axis=0):
     def _setitem_with_indexer(self, indexer, value):
         self.obj._set_values(indexer, value)
 
-
 def _check_bool_indexer(ax, key):
     # boolean indexing, need to check that the data are aligned, otherwise
     # disallowed
@@ -1010,7 +1009,6 @@ def _check_bool_indexer(ax, key):
     result = np.asarray(result, dtype=bool)
     return result
 
-
 def _is_series(obj):
     from pandas.core.series import Series
     return isinstance(obj, Series)

pandas/core/internals.py

@@ -80,7 +80,7 @@ def set_ref_locs(self, placement):
         if placement is None:
             self._ref_locs = None
         else:
-            self._ref_locs = np.array(placement,dtype='int64').copy()
+            self._ref_locs = np.array(placement,dtype='int64', copy=True)
 
     def set_ref_items(self, ref_items, maybe_rename=True):
         """
@@ -154,7 +154,8 @@ def reindex_axis(self, indexer, axis=1, fill_value=np.nan, mask_info=None):
             raise AssertionError('axis must be at least 1, got %d' % axis)
         new_values = com.take_nd(self.values, indexer, axis,
                                  fill_value=fill_value, mask_info=mask_info)
-        return make_block(new_values, self.items, self.ref_items, fastpath=True)
+        return make_block(new_values, self.items, self.ref_items, fastpath=True,
+                          placement=self._ref_locs)
 
     def reindex_items_from(self, new_ref_items, copy=True):
         """
@@ -168,6 +169,7 @@ def reindex_items_from(self, new_ref_items, copy=True):
         reindexed : Block
         """
         new_ref_items, indexer = self.items.reindex(new_ref_items)
+            
         if indexer is None:
             new_items = new_ref_items
             new_values = self.values.copy() if copy else self.values
@@ -1078,25 +1080,25 @@ def _set_ref_locs(self, labels=None, do_refs=False):
 
         """
 
-        im = None
         if labels is None:
             labels = self.items
-        else:
-            _ensure_index(labels)
 
         # we are unique, and coming from a unique
-        if labels.is_unique and not do_refs:
+        is_unique = labels.is_unique
+        if is_unique and not do_refs:
 
-            # reset our ref locs
-            self._ref_locs = None
-            for b in self.blocks:
-                b._ref_locs = None
+            if not self.items.is_unique:
+                
+                # reset our ref locs
+                self._ref_locs = None
+                for b in self.blocks:
+                    b._ref_locs = None
 
             return None
 
         # we are going to a non-unique index
         # we have ref_locs on the block at this point
-        if (not labels.is_unique and do_refs) or do_refs=='force':
+        if (not is_unique and do_refs) or do_refs=='force':
 
             # create the items map
             im = getattr(self,'_items_map',None)
@@ -1972,17 +1974,18 @@ def reindex_indexer(self, new_axis, indexer, axis=1, fill_value=np.nan):
     def _reindex_indexer_items(self, new_items, indexer, fill_value):
         # TODO: less efficient than I'd like
 
+        is_unique = self.items.is_unique
         item_order = com.take_1d(self.items.values, indexer)
 
         # keep track of what items aren't found anywhere
         mask = np.zeros(len(item_order), dtype=bool)
-
         new_axes = [new_items] + self.axes[1:]
 
         new_blocks = []
         for blk in self.blocks:
             blk_indexer = blk.items.get_indexer(item_order)
             selector = blk_indexer != -1
+
             # update with observed items
             mask |= selector
 
@@ -2142,7 +2145,7 @@ def rename_axis(self, mapper, axis=1):
 
     def rename_items(self, mapper, copydata=True):
         new_items = Index([mapper(x) for x in self.items])
-        new_items.is_unique
+        is_unique = new_items.is_unique
 
         new_blocks = []
         for block in self.blocks:

pandas/tests/test_frame.py

@@ -2860,6 +2860,12 @@ def check(result, expected=None):
         expected = DataFrame([[1,1,1,5,'bah',3],[1,1,2,5,'bah',3],[2,1,3,5,'bah',3]],columns=['foo','bar','foo','hello','string','foo2'])
         check(df,expected)
 
+        # set (non-dup)
+        df['foo2'] = 4
+        expected = DataFrame([[1,1,1,5,'bah',4],[1,1,2,5,'bah',4],[2,1,3,5,'bah',4]],columns=['foo','bar','foo','hello','string','foo2'])
+        check(df,expected)
+        df['foo2'] = 3
+
         # delete (non dup)
         del df['bar']
         expected = DataFrame([[1,1,5,'bah',3],[1,2,5,'bah',3],[2,3,5,'bah',3]],columns=['foo','foo','hello','string','foo2'])
@@ -2912,6 +2918,33 @@ def check(result, expected=None):
         expected = DataFrame([[1,5,7.],[1,5,7.],[1,5,7.]],columns=['bar','hello','foo2'])
         check(df,expected)
 
+        # reindex
+        df = DataFrame([[1,5,7.],[1,5,7.],[1,5,7.]],columns=['bar','a','a'])
+        expected = DataFrame([[1],[1],[1]],columns=['bar'])
+        result = df.reindex(columns=['bar'])
+        check(result,expected)
+
+        result1 = DataFrame([[1],[1],[1]],columns=['bar']).reindex(columns=['bar','foo'])
+        result2 = df.reindex(columns=['bar','foo'])
+        check(result2,result1)
+
+        # drop
+        df = DataFrame([[1,5,7.],[1,5,7.],[1,5,7.]],columns=['bar','a','a'])
+        df = df.drop(['a'],axis=1)
+        expected = DataFrame([[1],[1],[1]],columns=['bar'])
+        check(df,expected)
+
+    def test_insert_benchmark(self):
+        # from the vb_suite/frame_methods/frame_insert_columns
+        N = 10
+        K = 5
+        df = DataFrame(index=range(N))
+        new_col = np.random.randn(N)
+        for i in range(K):
+            df[i] = new_col
+        expected = DataFrame(np.repeat(new_col,K).reshape(N,K),index=range(N))
+        assert_frame_equal(df,expected)
+
     def test_constructor_single_value(self):
 
         # expecting single value upcasting here

pandas/tools/merge.py

@@ -781,10 +781,10 @@ def _upcast_blocks(blocks):
     for block in blocks:
         if isinstance(block, IntBlock):
             newb = make_block(block.values.astype(float), block.items,
-                              block.ref_items)
+                              block.ref_items, placement=block._ref_locs)
         elif isinstance(block, BoolBlock):
             newb = make_block(block.values.astype(object), block.items,
-                              block.ref_items)
+                              block.ref_items, placement=block._ref_locs)
         else:
             newb = block
         new_blocks.append(newb)