Python 3 compatibility

pandas/core/common.py

@@ -1,8 +1,10 @@
 """
 Misc tools for implementing data structures
 """
-
-from cStringIO import StringIO
+try:
+    from io import BytesIO
+except ImportError:   # Python < 2.6
+    from cStringIO import StringIO as BytesIO
 import itertools
 
 from numpy.lib.format import read_array, write_array
@@ -76,13 +78,13 @@ def notnull(obj):
 def _pickle_array(arr):
     arr = arr.view(np.ndarray)
 
-    buf = StringIO()
+    buf = BytesIO()
     write_array(buf, arr)
 
     return buf.getvalue()
 
 def _unpickle_array(bytes):
-    arr = read_array(StringIO(bytes))
+    arr = read_array(BytesIO(bytes))
     return arr
 
 def _take_1d_bool(arr, indexer, out):
@@ -390,7 +392,7 @@ def rands(n):
     """Generates a random alphanumeric string of length *n*"""
     from random import Random
     import string
-    return ''.join(Random().sample(string.letters+string.digits, n))
+    return ''.join(Random().sample(string.ascii_letters+string.digits, n))
 
 def adjoin(space, *lists):
     """
@@ -458,7 +460,11 @@ def __init__(self, seq, key=lambda x:x):
         for value in seq:
             k = key(value)
             self.setdefault(k, []).append(value)
-    __iter__ = dict.iteritems
+    try:
+        __iter__ = dict.iteritems
+    except AttributeError:  # Python 3
+        def __iter__(self):
+            return iter(dict.items(self))
 
 def map_indices_py(arr):
     """

pandas/core/frame.py

@@ -29,6 +29,7 @@
 from pandas.core.internals import BlockManager, make_block, form_blocks
 from pandas.core.series import Series, _is_bool_indexer
 from pandas.util.decorators import deprecate
+from pandas.util import py3compat
 import pandas.core.common as common
 import pandas.core.datetools as datetools
 import pandas._tseries as _tseries
@@ -276,6 +277,10 @@ def iteritems(self):
         """Iterator over (column, series) pairs"""
         series = self._series
         return ((k, series[k]) for k in self.columns)
+
+    iterkv = iteritems
+    if py3compat.PY3:
+        items = iteritems
 
     def __len__(self):
         """Returns length of index"""
@@ -291,7 +296,7 @@ def __contains__(self, key):
     add = _arith_method(operator.add, 'add')
     mul = _arith_method(operator.mul, 'multiply')
     sub = _arith_method(operator.sub, 'subtract')
-    div = _arith_method(operator.div, 'divide')
+    div = _arith_method(lambda x, y: x / y, 'divide')
 
     radd = _arith_method(operator.add, 'add')
     rmul = _arith_method(operator.mul, 'multiply')
@@ -301,19 +306,26 @@ def __contains__(self, key):
     __add__ = _arith_method(operator.add, '__add__', default_axis=None)
     __sub__ = _arith_method(operator.sub, '__sub__', default_axis=None)
     __mul__ = _arith_method(operator.mul, '__mul__', default_axis=None)
-    __div__ = _arith_method(operator.div, '__div__', default_axis=None)
     __truediv__ = _arith_method(operator.truediv, '__truediv__',
                                default_axis=None)
+    __floordiv__ = _arith_method(operator.floordiv, '__floordiv__',
+                               default_axis=None)
     __pow__ = _arith_method(operator.pow, '__pow__', default_axis=None)
 
     __radd__ = _arith_method(operator.add, '__radd__', default_axis=None)
     __rmul__ = _arith_method(operator.mul, '__rmul__', default_axis=None)
     __rsub__ = _arith_method(lambda x, y: y - x, '__rsub__', default_axis=None)
-    __rdiv__ = _arith_method(lambda x, y: y / x, '__rdiv__', default_axis=None)
     __rtruediv__ = _arith_method(lambda x, y: y / x, '__rtruediv__',
                                 default_axis=None)
+    __rfloordiv__ = _arith_method(lambda x, y: y // x, '__rfloordiv__',
+                               default_axis=None)
     __rpow__ = _arith_method(lambda x, y: y ** x, '__rpow__',
                              default_axis=None)
+
+    # Python 2 division methods
+    if not py3compat.PY3:
+        __div__ = _arith_method(operator.div, '__div__', default_axis=None)
+        __rdiv__ = _arith_method(lambda x, y: y / x, '__rdiv__', default_axis=None)
 
     def __neg__(self):
         return self * -1
@@ -463,7 +475,7 @@ def to_sparse(self, fill_value=None, kind='block'):
                                default_fill_value=fill_value)
 
     def to_csv(self, path, nanRep='', cols=None, header=True,
-              index=True, index_label=None, mode='wb'):
+              index=True, index_label=None, mode='w'):
         """
         Write DataFrame to a comma-separated values (csv) file
 
@@ -482,7 +494,7 @@ def to_csv(self, path, nanRep='', cols=None, header=True,
             Column label for index column(s) if desired. If None is given, and
             `header` and `index` are True, then the index names are used. A
             sequence should be given if the DataFrame uses MultiIndex.
-        mode : Python write mode, default 'wb'
+        mode : Python write mode, default 'w'
         """
         f = open(path, mode)
 
@@ -652,7 +664,7 @@ def dtypes(self):
 
     def get_dtype_counts(self):
         counts = {}
-        for _, series in self.iteritems():
+        for _, series in self.iterkv():
             if series.dtype in counts:
                 counts[series.dtype] += 1
             else:
@@ -909,7 +921,7 @@ def _set_item(self, key, value):
     def _sanitize_column(self, value):
         # Need to make sure new columns (which go into the BlockManager as new
         # blocks) are always copied
-        if hasattr(value, '__iter__'):
+        if hasattr(value, '__iter__') and not isinstance(value, basestring):
             if isinstance(value, Series):
                 if value.index.equals(self.index):
                     # copy the values

pandas/core/panel.py

@@ -19,6 +19,7 @@
 from pandas.core.generic import AxisProperty, NDFrame
 from pandas.core.series import Series
 from pandas.util.decorators import deprecate
+from pandas.util import py3compat
 import pandas.core.common as common
 import pandas._tseries as _tseries
 
@@ -174,16 +175,22 @@ class Panel(NDFrame):
 
     __add__ = _arith_method(operator.add, '__add__')
     __sub__ = _arith_method(operator.sub, '__sub__')
+    __truediv__ = _arith_method(operator.truediv, '__truediv__')
+    __floordiv__ = _arith_method(operator.floordiv, '__floordiv__')
     __mul__ = _arith_method(operator.mul, '__mul__')
-    __div__ = _arith_method(operator.div, '__div__')
     __pow__ = _arith_method(operator.pow, '__pow__')
 
     __radd__ = _arith_method(operator.add, '__radd__')
     __rmul__ = _arith_method(operator.mul, '__rmul__')
     __rsub__ = _arith_method(lambda x, y: y - x, '__rsub__')
-    __rdiv__ = _arith_method(lambda x, y: y / x, '__rdiv__')
+    __rtruediv__ = _arith_method(lambda x, y: y / x, '__rtruediv__')
+    __rfloordiv__ = _arith_method(lambda x, y: y // x, '__rfloordiv__')
     __rpow__ = _arith_method(lambda x, y: y ** x, '__rpow__')
 
+    if not py3compat.PY3:
+        __div__ = _arith_method(operator.div, '__div__')
+        __rdiv__ = _arith_method(lambda x, y: y / x, '__rdiv__')
+
     def __init__(self, data, items=None, major_axis=None, minor_axis=None,
                  copy=False, dtype=None):
         """
@@ -333,6 +340,10 @@ def __iter__(self):
     def iteritems(self):
         for item in self.items:
             yield item, self[item]
+
+    # Name that won't get automatically converted to items by 2to3. items is
+    # already in use for the first axis.
+    iterkv = iteritems
 
     def _get_plane_axes(self, axis):
         """
@@ -387,7 +398,7 @@ def to_sparse(self, fill_value=None, kind='block'):
         y : SparseDataFrame
         """
         from pandas.core.sparse import SparsePanel
-        frames = dict(self.iteritems())
+        frames = dict(self.iterkv())
         return SparsePanel(frames, items=self.items,
                            major_axis=self.major_axis,
                            minor_axis=self.minor_axis,
@@ -636,7 +647,7 @@ def fillna(self, value=None, method='pad'):
         """
         if value is None:
             result = {}
-            for col, s in self.iteritems():
+            for col, s in self.iterkv():
                 result[col] = s.fillna(method=method, value=value)
 
             return Panel.from_dict(result)
@@ -646,8 +657,12 @@ def fillna(self, value=None, method='pad'):
 
     add = _panel_arith_method(operator.add, 'add')
     subtract = sub = _panel_arith_method(operator.sub, 'subtract')
-    divide = div = _panel_arith_method(operator.div, 'divide')
     multiply = mul = _panel_arith_method(operator.mul, 'multiply')
+
+    try:
+        divide = div = _panel_arith_method(operator.div, 'divide')
+    except AttributeError:   # Python 3
+        divide = div = _panel_arith_method(operator.truediv, 'divide')
 
     def major_xs(self, key, copy=True):
         """
@@ -1218,8 +1233,12 @@ def _combine_panel_frame(self, other, func, axis='items'):
 
     add = _panel_arith_method(operator.add, 'add')
     subtract = sub = _panel_arith_method(operator.sub, 'subtract')
-    divide = div = _panel_arith_method(operator.div, 'divide')
     multiply = mul = _panel_arith_method(operator.mul, 'multiply')
+
+    try:
+        divide = div = _panel_arith_method(operator.div, 'divide')
+    except AttributeError:   # Python 3
+        divide = div = _panel_arith_method(operator.truediv, 'divide')
 
     def to_wide(self):
         """

pandas/core/series.py

@@ -20,6 +20,7 @@
 from pandas.core.index import Index, MultiIndex, _ensure_index
 from pandas.core.indexing import _SeriesIndexer, _maybe_droplevels
 from pandas.util.decorators import deprecate
+from pandas.util import py3compat
 import pandas.core.common as common
 import pandas.core.datetools as datetools
 import pandas._tseries as lib
@@ -366,31 +367,41 @@ def iteritems(self):
         Lazily iterate over (index, value) tuples
         """
         return itertools.izip(iter(self.index), iter(self))
+
+    iterkv = iteritems
+    if py3compat.PY3:
+        items = iteritems
 
     #----------------------------------------------------------------------
     #   Arithmetic operators
 
     __add__ = _arith_method(operator.add, '__add__')
     __sub__ = _arith_method(operator.sub, '__sub__')
     __mul__ = _arith_method(operator.mul, '__mul__')
-    __div__ = _arith_method(operator.div, '__div__')
     __truediv__ = _arith_method(operator.truediv, '__truediv__')
+    __floordiv__ = _arith_method(operator.floordiv, '__floordiv__')
     __pow__ = _arith_method(operator.pow, '__pow__')
-    __truediv__ = _arith_method(operator.truediv, '__truediv__')
 
     __radd__ = _arith_method(operator.add, '__add__')
     __rmul__ = _arith_method(operator.mul, '__mul__')
     __rsub__ = _arith_method(lambda x, y: y - x, '__sub__')
-    __rdiv__ = _arith_method(lambda x, y: y / x, '__div__')
     __rtruediv__ = _arith_method(lambda x, y: y / x, '__truediv__')
+    __rfloordiv__ = _arith_method(lambda x, y: y // x, '__floordiv__')
     __rpow__ = _arith_method(lambda x, y: y ** x, '__pow__')
 
     # Inplace operators
     __iadd__ = __add__
     __isub__ = __sub__
     __imul__ = __mul__
-    __idiv__ = __div__
+    __itruediv__ = __truediv__
+    __ifloordiv__ = __floordiv__
     __ipow__ = __pow__
+
+    # Python 2 division operators
+    if not py3compat.PY3:
+        __div__ = _arith_method(operator.div, '__div__')
+        __rdiv__ = _arith_method(lambda x, y: y / x, '__div__')
+        __idiv__ = __div__
 
     #----------------------------------------------------------------------
     # Misc public methods
@@ -977,7 +988,10 @@ def _binop(self, other, func, fill_value=None):
     add = _flex_method(operator.add, 'add')
     sub = _flex_method(operator.sub, 'subtract')
     mul = _flex_method(operator.mul, 'multiply')
-    div = _flex_method(operator.div, 'divide')
+    try:
+        div = _flex_method(operator.div, 'divide')
+    except AttributeError:    # Python 3
+        div = _flex_method(operator.truediv, 'divide')
 
     def combine(self, other, func, fill_value=nan):
         """
@@ -1529,7 +1543,7 @@ def to_csv(self, path):
         path : string or None
             Output filepath. If None, write to stdout
         """
-        f = open(path, 'wb')
+        f = open(path, 'w')
         for idx, value in self.iteritems():
             f.write(str(idx) + ',' + str(value) + '\n')
         f.close()

pandas/core/sparse.py

@@ -20,6 +20,8 @@
 import pandas.core.common as common
 import pandas.core.datetools as datetools
 
+from pandas.util import py3compat
+
 from pandas._sparse import BlockIndex, IntIndex
 import pandas._sparse as splib
 
@@ -286,24 +288,31 @@ def __repr__(self):
     __add__ = _sparse_op_wrap(operator.add, 'add')
     __sub__ = _sparse_op_wrap(operator.sub, 'sub')
     __mul__ = _sparse_op_wrap(operator.mul, 'mul')
-    __div__ = _sparse_op_wrap(operator.div, 'div')
     __truediv__ = _sparse_op_wrap(operator.truediv, 'truediv')
+    __floordiv__ = _sparse_op_wrap(operator.floordiv, 'floordiv')
     __pow__ = _sparse_op_wrap(operator.pow, 'pow')
 
     # reverse operators
     __radd__ = _sparse_op_wrap(operator.add, '__radd__')
-    __rmul__ = _sparse_op_wrap(operator.mul, '__rmul__')
     __rsub__ = _sparse_op_wrap(lambda x, y: y - x, '__rsub__')
-    __rdiv__ = _sparse_op_wrap(lambda x, y: y / x, '__rdiv__')
+    __rmul__ = _sparse_op_wrap(operator.mul, '__rmul__')
     __rtruediv__ = _sparse_op_wrap(lambda x, y: y / x, '__rtruediv__')
+    __rfloordiv__ = _sparse_op_wrap(lambda x, y: y // x, 'floordiv')
     __rpow__ = _sparse_op_wrap(lambda x, y: y ** x, '__rpow__')
 
     # Inplace operators
     __iadd__ = __add__
     __isub__ = __sub__
     __imul__ = __mul__
-    __idiv__ = __div__
+    __itruediv__ = __truediv__
+    __ifloordiv__ = __floordiv__
     __ipow__ = __pow__
+
+    # Python 2 division operators
+    if not py3compat.PY3:
+        __div__ = _sparse_op_wrap(operator.div, 'div')
+        __rdiv__ = _sparse_op_wrap(lambda x, y: y / x, '__rdiv__')
+        __idiv__ = __div__
 
     @property
     def values(self):
@@ -1555,7 +1564,7 @@ def _combine(self, other, func, axis=0):
             return self._combinePanel(other, func)
         elif np.isscalar(other):
             new_frames = dict((k, func(v, other))
-                              for k, v in self.iteritems())
+                              for k, v in self.iterkv())
             return self._new_like(new_frames)
 
     def _combineFrame(self, other, func, axis=0):
@@ -1631,7 +1640,7 @@ def major_xs(self, key):
         y : DataFrame
             index -> minor axis, columns -> items
         """
-        slices = dict((k, v.xs(key)) for k, v in self.iteritems())
+        slices = dict((k, v.xs(key)) for k, v in self.iterkv())
         return DataFrame(slices, index=self.minor_axis, columns=self.items)
 
     def minor_xs(self, key):
@@ -1648,7 +1657,7 @@ def minor_xs(self, key):
         y : SparseDataFrame
             index -> major axis, columns -> items
         """
-        slices = dict((k, v[key]) for k, v in self.iteritems())
+        slices = dict((k, v[key]) for k, v in self.iterkv())
         return SparseDataFrame(slices, index=self.major_axis,
                                columns=self.items,
                                default_fill_value=self.default_fill_value,

pandas/io/tests/test_parsers.py

@@ -120,6 +120,11 @@ def test_read_csv_no_index_name(self):
         assert_frame_equal(df, df2)
 
     def test_excel_table(self):
+        try:
+            import xlrd
+        except ImportError:
+            import nose
+            raise nose.SkipTest
         pth = os.path.join(self.dirpath, 'test.xls')
         xls = ExcelFile(pth)
         df = xls.parse('Sheet1')