PERF: Perf issue in concatting with empty objects (GH3259)

Author: jreback

doc/source/release.rst

@@ -166,6 +166,7 @@ Bug Fixes
 - Bug in ``Series.reindex`` when specifying a ``method`` with some nan values was inconsistent (noted on a resample) (:issue:`6418`)
 - Bug in :meth:`DataFrame.replace` where nested dicts were erroneously
   depending on the order of dictionary keys and values (:issue:`5338`).
+- Perf issue in concatting with empty objects (:issue:`3259`)
 
 pandas 0.13.1
 -------------

pandas/core/groupby.py

@@ -2209,10 +2209,7 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False):
 
             # make Nones an empty object
             if com._count_not_none(*values) != len(values):
-                v = None
-                for v in values:
-                    if v is not None:
-                        break
+                v = next(v for v in values if v is not None)
                 if v is None:
                     return DataFrame()
                 elif isinstance(v, NDFrame):

pandas/tools/merge.py

@@ -957,7 +957,7 @@ def __init__(self, objs, axis=0, join='outer', join_axes=None,
             objs = [objs[k] for k in keys]
 
         if keys is None:
-            objs = [obj for obj in objs if obj is not None]
+            objs = [obj for obj in objs if obj is not None ]
         else:
             # #1649
             clean_keys = []
@@ -970,16 +970,25 @@ def __init__(self, objs, axis=0, join='outer', join_axes=None,
             objs = clean_objs
             keys = clean_keys
 
-        if len(objs) == 0:
-            raise Exception('All objects passed were None')
-
         # consolidate data
+        self.objs = []
         for obj in objs:
-            if isinstance(obj, NDFrame):
-                obj.consolidate(inplace=True)
-        self.objs = objs
+            if not isinstance(obj, NDFrame):
+                raise TypeError("cannot concatenate a non-NDFrame object")
+
+            # skip completely empty
+            if not np.sum(obj.shape):
+                continue
+
+            # consolidate
+            obj.consolidate(inplace=True)
+            self.objs.append(obj)
+
+        if len(self.objs) == 0:
+            raise Exception('All objects passed were None')
 
-        sample = objs[0]
+        # need the first as a sample non-empty as a sample
+        sample = next(obj for obj in self.objs if np.prod(obj.shape))
 
         # Need to flip BlockManager axis in the DataFrame special case
         if isinstance(sample, DataFrame):

pandas/tools/tests/test_merge.py

@@ -1653,6 +1653,15 @@ def test_handle_empty_objects(self):
 
         tm.assert_frame_equal(concatted, expected)
 
+        # empty as first element with time series
+        # GH3259
+        df = DataFrame(dict(A = range(10000)),index=date_range('20130101',periods=10000,freq='s'))
+        empty = DataFrame()
+        result = concat([df,empty])
+        assert_frame_equal(result, df)
+        result = concat([empty,df])
+        assert_frame_equal(result, df)
+
     def test_panel_join(self):
         panel = tm.makePanel()
         tm.add_nans(panel)
@@ -1967,6 +1976,13 @@ def test_concat_series_axis1_same_names_ignore_index(self):
         result = concat([s1, s2], axis=1, ignore_index=True)
         self.assertTrue(np.array_equal(result.columns, [0, 1]))
 
+    def test_concat_invalid(self):
+
+        # trying to concat a ndframe with a non-ndframe
+        df1 = mkdf(10, 2)
+        for obj in [1, dict(), [1, 2], (1, 2) ]:
+            self.assertRaises(TypeError, lambda x: concat([ df1, obj ]))
+
     def test_concat_invalid_first_argument(self):
         df1 = mkdf(10, 2)
         df2 = mkdf(10, 2)

vb_suite/join_merge.py

@@ -186,6 +186,21 @@ def sample(values, k):
 concat_small_frames = Benchmark('concat([df] * 1000)', setup,
                                 start_date=datetime(2012, 1, 1))
 
+
+#----------------------------------------------------------------------
+# Concat empty
+
+setup = common_setup + """
+df = DataFrame(dict(A = range(10000)),index=date_range('20130101',periods=10000,freq='s'))
+empty = DataFrame()
+"""
+
+concat_empty_frames1 = Benchmark('concat([df,empty)', setup,
+                                start_date=datetime(2012, 1, 1))
+concat_empty_frames2 = Benchmark('concat([empty,df)', setup,
+                                start_date=datetime(2012, 1, 1))
+
+
 #----------------------------------------------------------------------
 # Ordered merge