BUG: Fix index for datetime64 conversion. Fixes #13937

closes #13937

Author: Amol Kahat <[email protected]>

Closes #14446 from amolkahat/bug_fixes and squashes the following commits:

3806983 [Amol Kahat] Modify test cases.

Author: amolkahat

doc/source/whatsnew/v0.20.0.txt

@@ -612,6 +612,7 @@ Bug Fixes
 - Bug in ``GroupBy.get_group()`` failing with a categorical grouper (:issue:`15155`)
 - Bug in ``pandas.tools.utils.cartesian_product()`` with large input can cause overflow on windows (:issue:`15265`)
 
+- Bug in ``DataFrame.to_records()`` with converting a ``DatetimeIndex`` with a timezone (:issue:`13937`)
 
 
 - Bug in ``.groupby(...).rolling(...)`` when ``on`` is specified and using a ``DatetimeIndex`` (:issue:`15130`)

pandas/core/frame.py

@@ -36,7 +36,7 @@
                                  is_object_dtype,
                                  is_extension_type,
                                  is_datetimetz,
-                                 is_datetime64_dtype,
+                                 is_datetime64_any_dtype,
                                  is_datetime64tz_dtype,
                                  is_bool_dtype,
                                  is_integer_dtype,
@@ -1103,7 +1103,7 @@ def to_records(self, index=True, convert_datetime64=True):
         y : recarray
         """
         if index:
-            if is_datetime64_dtype(self.index) and convert_datetime64:
+            if is_datetime64_any_dtype(self.index) and convert_datetime64:
                 ix_vals = [self.index.to_pydatetime()]
             else:
                 if isinstance(self.index, MultiIndex):

pandas/tests/frame/test_convert_to.py

@@ -1,16 +1,13 @@
 # -*- coding: utf-8 -*-
 
-from __future__ import print_function
-
-from numpy import nan
+import pytest
 import numpy as np
 
 from pandas import compat
 from pandas import (DataFrame, Series, MultiIndex, Timestamp,
                     date_range)
 
 import pandas.util.testing as tm
-
 from pandas.tests.frame.common import TestData
 
 
@@ -41,13 +38,13 @@ def test_to_dict(self):
 
         recons_data = DataFrame(test_data).to_dict("sp")
         expected_split = {'columns': ['A', 'B'], 'index': ['1', '2', '3'],
-                          'data': [[1.0, '1'], [2.0, '2'], [nan, '3']]}
+                          'data': [[1.0, '1'], [2.0, '2'], [np.nan, '3']]}
         tm.assert_dict_equal(recons_data, expected_split)
 
         recons_data = DataFrame(test_data).to_dict("r")
         expected_records = [{'A': 1.0, 'B': '1'},
                             {'A': 2.0, 'B': '2'},
-                            {'A': nan, 'B': '3'}]
+                            {'A': np.nan, 'B': '3'}]
         tm.assertIsInstance(recons_data, list)
         self.assertEqual(len(recons_data), 3)
         for l, r in zip(recons_data, expected_records):
@@ -192,3 +189,18 @@ def test_to_records_with_unicode_column_names(self):
                    "formats": ['<i8', '<f8']}
         )
         tm.assert_almost_equal(result, expected)
+
+
+@pytest.mark.parametrize('tz', ['UTC', 'GMT', 'US/Eastern'])
+def test_to_records_datetimeindex_with_tz(tz):
+    # GH13937
+    dr = date_range('2016-01-01', periods=10,
+                    freq='S', tz=tz)
+
+    df = DataFrame({'datetime': dr}, index=dr)
+
+    expected = df.to_records()
+    result = df.tz_convert("UTC").to_records()
+
+    # both converted to UTC, so they are equal
+    tm.assert_numpy_array_equal(result, expected)