Merge remote-tracking branch 'upstream/master' into STY-repr-batch-3

Author: MomIsBestFriend

ci/deps/azure-windows-36.yaml

@@ -20,6 +20,7 @@ dependencies:
   - numexpr
   - numpy=1.15.*
   - openpyxl
+  - jinja2
   - pyarrow>=0.12.0
   - pytables
   - python-dateutil

pandas/io/pytables.py

@@ -58,7 +58,6 @@
 import pandas.core.common as com
 from pandas.core.computation.pytables import PyTablesExpr, maybe_expression
 from pandas.core.index import ensure_index
-from pandas.core.internals import BlockManager, _block_shape, make_block
 
 from pandas.io.common import _stringify_path
 from pandas.io.formats.printing import adjoin, pprint_thing
@@ -2392,7 +2391,7 @@ def set_atom_categorical(self, block, items, info=None):
         # write the codes; must be in a block shape
         self.ordered = values.ordered
         self.typ = self.get_atom_data(block, kind=codes.dtype.name)
-        self.set_data(_block_shape(codes))
+        self.set_data(codes)
 
         # write the categories
         self.meta = "category"
@@ -3186,17 +3185,23 @@ def read(
             axes.append(ax)
 
         items = axes[0]
-        blocks = []
+        dfs = []
+
         for i in range(self.nblocks):
 
             blk_items = self.read_index(f"block{i}_items")
             values = self.read_array(f"block{i}_values", start=_start, stop=_stop)
-            blk = make_block(
-                values, placement=items.get_indexer(blk_items), ndim=len(axes)
-            )
-            blocks.append(blk)
 
-        return self.obj_type(BlockManager(blocks, axes))
+            columns = items[items.get_indexer(blk_items)]
+            df = DataFrame(values.T, columns=columns, index=axes[1])
+            dfs.append(df)
+
+        if len(dfs) > 0:
+            out = concat(dfs, axis=1)
+            out = out.reindex(columns=items, copy=False)
+            return out
+
+        return DataFrame(columns=axes[0], index=axes[1])
 
     def write(self, obj, **kwargs):
         super().write(obj, **kwargs)
@@ -4432,9 +4437,15 @@ def read(
             if values.ndim == 1 and isinstance(values, np.ndarray):
                 values = values.reshape((1, values.shape[0]))
 
-            block = make_block(values, placement=np.arange(len(cols_)), ndim=2)
-            mgr = BlockManager([block], [cols_, index_])
-            frames.append(DataFrame(mgr))
+            if isinstance(values, np.ndarray):
+                df = DataFrame(values.T, columns=cols_, index=index_)
+            elif isinstance(values, Index):
+                df = DataFrame(values, columns=cols_, index=index_)
+            else:
+                # Categorical
+                df = DataFrame([values], columns=cols_, index=index_)
+            assert (df.dtypes == values.dtype).all(), (df.dtypes, values.dtype)
+            frames.append(df)
 
         if len(frames) == 1:
             df = frames[0]

pandas/tests/arithmetic/test_datetime64.py

@@ -171,9 +171,9 @@ class TestDatetime64SeriesComparison:
         ],
     )
     @pytest.mark.parametrize("reverse", [True, False])
-    @pytest.mark.parametrize("box", [Series, pd.Index])
     @pytest.mark.parametrize("dtype", [None, object])
-    def test_nat_comparisons(self, dtype, box, reverse, pair):
+    def test_nat_comparisons(self, dtype, index_or_series, reverse, pair):
+        box = index_or_series
         l, r = pair
         if reverse:
             # add lhs / rhs switched data
@@ -2383,14 +2383,16 @@ def test_dti_add_series(self, tz, names):
         result4 = index + ser.values
         tm.assert_index_equal(result4, expected)
 
-    @pytest.mark.parametrize("other_box", [pd.Index, Series])
     @pytest.mark.parametrize("op", [operator.add, roperator.radd, operator.sub])
     @pytest.mark.parametrize(
         "names", [(None, None, None), ("foo", "bar", None), ("foo", "foo", "foo")]
     )
-    def test_dti_addsub_offset_arraylike(self, tz_naive_fixture, names, op, other_box):
+    def test_dti_addsub_offset_arraylike(
+        self, tz_naive_fixture, names, op, index_or_series
+    ):
         # GH#18849, GH#19744
         box = pd.Index
+        other_box = index_or_series
         from .test_timedelta64 import get_upcast_box
 
         tz = tz_naive_fixture

pandas/tests/frame/test_dtypes.py

@@ -4,7 +4,7 @@
 import numpy as np
 import pytest
 
-from pandas.core.dtypes.dtypes import CategoricalDtype, DatetimeTZDtype
+from pandas.core.dtypes.dtypes import CategoricalDtype, DatetimeTZDtype, IntervalDtype
 
 import pandas as pd
 from pandas import (
@@ -699,14 +699,7 @@ def test_astype_categorical(self, dtype):
         expected = DataFrame({k: Categorical(d[k], dtype=dtype) for k in d})
         tm.assert_frame_equal(result, expected)
 
-    @pytest.mark.parametrize(
-        "cls",
-        [
-            pd.api.types.CategoricalDtype,
-            pd.api.types.DatetimeTZDtype,
-            pd.api.types.IntervalDtype,
-        ],
-    )
+    @pytest.mark.parametrize("cls", [CategoricalDtype, DatetimeTZDtype, IntervalDtype])
     def test_astype_categoricaldtype_class_raises(self, cls):
         df = DataFrame({"A": ["a", "a", "b", "c"]})
         xpr = "Expected an instance of {}".format(cls.__name__)

pandas/tests/test_downstream.py

@@ -102,10 +102,7 @@ def test_pandas_datareader():
 
 
 # importing from pandas, Cython import warning
-@pytest.mark.filterwarnings("ignore:The 'warn':DeprecationWarning")
-@pytest.mark.filterwarnings("ignore:pandas.util:DeprecationWarning")
 @pytest.mark.filterwarnings("ignore:can't resolve:ImportWarning")
-@pytest.mark.skip(reason="gh-25778: geopandas stack issue")
 def test_geopandas():
 
     geopandas = import_module("geopandas")  # noqa

pandas/tests/test_nanops.py

@@ -24,7 +24,7 @@ def setup_method(self, method):
         np.random.seed(11235)
         nanops._USE_BOTTLENECK = False
 
-        arr_shape = (11, 7, 5)
+        arr_shape = (11, 7)
 
         self.arr_float = np.random.randn(*arr_shape)
         self.arr_float1 = np.random.randn(*arr_shape)
@@ -68,21 +68,21 @@ def setup_method(self, method):
             self.arr_nan_infj = self.arr_inf * 1j
             self.arr_complex_nan_infj = np.vstack([self.arr_complex, self.arr_nan_infj])
 
-        self.arr_float_2d = self.arr_float[:, :, 0]
-        self.arr_float1_2d = self.arr_float1[:, :, 0]
+        self.arr_float_2d = self.arr_float
+        self.arr_float1_2d = self.arr_float1
 
-        self.arr_nan_2d = self.arr_nan[:, :, 0]
-        self.arr_float_nan_2d = self.arr_float_nan[:, :, 0]
-        self.arr_float1_nan_2d = self.arr_float1_nan[:, :, 0]
-        self.arr_nan_float1_2d = self.arr_nan_float1[:, :, 0]
+        self.arr_nan_2d = self.arr_nan
+        self.arr_float_nan_2d = self.arr_float_nan
+        self.arr_float1_nan_2d = self.arr_float1_nan
+        self.arr_nan_float1_2d = self.arr_nan_float1
 
-        self.arr_float_1d = self.arr_float[:, 0, 0]
-        self.arr_float1_1d = self.arr_float1[:, 0, 0]
+        self.arr_float_1d = self.arr_float[:, 0]
+        self.arr_float1_1d = self.arr_float1[:, 0]
 
-        self.arr_nan_1d = self.arr_nan[:, 0, 0]
-        self.arr_float_nan_1d = self.arr_float_nan[:, 0, 0]
-        self.arr_float1_nan_1d = self.arr_float1_nan[:, 0, 0]
-        self.arr_nan_float1_1d = self.arr_nan_float1[:, 0, 0]
+        self.arr_nan_1d = self.arr_nan[:, 0]
+        self.arr_float_nan_1d = self.arr_float_nan[:, 0]
+        self.arr_float1_nan_1d = self.arr_float1_nan[:, 0]
+        self.arr_nan_float1_1d = self.arr_nan_float1[:, 0]
 
     def teardown_method(self, method):
         nanops._USE_BOTTLENECK = use_bn

pandas/tseries/frequencies.py

@@ -1,6 +1,6 @@
 from datetime import timedelta
 import re
-from typing import Dict
+from typing import Dict, Optional
 
 import numpy as np
 from pytz import AmbiguousTimeError
@@ -52,8 +52,10 @@
 _offset_map: Dict[str, DateOffset] = {}
 
 
-def get_period_alias(offset_str):
-    """ alias to closest period strings BQ->Q etc"""
+def get_period_alias(offset_str: str) -> Optional[str]:
+    """
+    Alias to closest period strings BQ->Q etc.
+    """
     return _offset_to_period_map.get(offset_str, None)
 
 
@@ -68,7 +70,7 @@ def get_period_alias(offset_str):
 }
 
 
-def to_offset(freq):
+def to_offset(freq) -> Optional[DateOffset]:
     """
     Return DateOffset object from string or tuple representation
     or datetime.timedelta object.
@@ -179,9 +181,9 @@ def to_offset(freq):
     return delta
 
 
-def get_offset(name):
+def get_offset(name: str) -> DateOffset:
     """
-    Return DateOffset object associated with rule name
+    Return DateOffset object associated with rule name.
 
     Examples
     --------
@@ -214,7 +216,7 @@ def get_offset(name):
 # Period codes
 
 
-def infer_freq(index, warn=True):
+def infer_freq(index, warn: bool = True) -> Optional[str]:
     """
     Infer the most likely frequency given the input index. If the frequency is
     uncertain, a warning will be printed.
@@ -247,6 +249,7 @@ def infer_freq(index, warn=True):
             )
         index = values
 
+    inferer: _FrequencyInferer
     if is_period_arraylike(index):
         raise TypeError(
             "PeriodIndex given. Check the `freq` attribute "
@@ -280,7 +283,7 @@ class _FrequencyInferer:
     Not sure if I can avoid the state machine here
     """
 
-    def __init__(self, index, warn=True):
+    def __init__(self, index, warn: bool = True):
         self.index = index
         self.values = index.asi8
 
@@ -315,7 +318,7 @@ def is_unique(self) -> bool:
     def is_unique_asi8(self):
         return len(self.deltas_asi8) == 1
 
-    def get_freq(self):
+    def get_freq(self) -> Optional[str]:
         """
         Find the appropriate frequency string to describe the inferred
         frequency of self.values
@@ -388,7 +391,7 @@ def mdiffs(self):
     def ydiffs(self):
         return unique_deltas(self.fields["Y"].astype("i8"))
 
-    def _infer_daily_rule(self):
+    def _infer_daily_rule(self) -> Optional[str]:
         annual_rule = self._get_annual_rule()
         if annual_rule:
             nyears = self.ydiffs[0]
@@ -424,7 +427,9 @@ def _infer_daily_rule(self):
         if wom_rule:
             return wom_rule
 
-    def _get_annual_rule(self):
+        return None
+
+    def _get_annual_rule(self) -> Optional[str]:
         if len(self.ydiffs) > 1:
             return None
 
@@ -434,7 +439,7 @@ def _get_annual_rule(self):
         pos_check = self.month_position_check()
         return {"cs": "AS", "bs": "BAS", "ce": "A", "be": "BA"}.get(pos_check)
 
-    def _get_quarterly_rule(self):
+    def _get_quarterly_rule(self) -> Optional[str]:
         if len(self.mdiffs) > 1:
             return None
 
@@ -444,13 +449,13 @@ def _get_quarterly_rule(self):
         pos_check = self.month_position_check()
         return {"cs": "QS", "bs": "BQS", "ce": "Q", "be": "BQ"}.get(pos_check)
 
-    def _get_monthly_rule(self):
+    def _get_monthly_rule(self) -> Optional[str]:
         if len(self.mdiffs) > 1:
             return None
         pos_check = self.month_position_check()
         return {"cs": "MS", "bs": "BMS", "ce": "M", "be": "BM"}.get(pos_check)
 
-    def _is_business_daily(self):
+    def _is_business_daily(self) -> bool:
         # quick check: cannot be business daily
         if self.day_deltas != [1, 3]:
             return False
@@ -465,7 +470,7 @@ def _is_business_daily(self):
             | ((weekdays > 0) & (weekdays <= 4) & (shifts == 1))
         )
 
-    def _get_wom_rule(self):
+    def _get_wom_rule(self) -> Optional[str]:
         #         wdiffs = unique(np.diff(self.index.week))
         # We also need -47, -49, -48 to catch index spanning year boundary
         #     if not lib.ismember(wdiffs, set([4, 5, -47, -49, -48])).all():
@@ -501,11 +506,11 @@ def _infer_daily_rule(self):
                 return _maybe_add_count("D", days)
 
 
-def _is_multiple(us, mult):
+def _is_multiple(us, mult: int) -> bool:
     return us % mult == 0
 
 
-def _maybe_add_count(base, count):
+def _maybe_add_count(base: str, count: float) -> str:
     if count != 1:
         assert count == int(count)
         count = int(count)