Merge remote-tracking branch 'upstream/master' into 28547

Author: cruzzoe

doc/source/whatsnew/v1.0.0.rst

@@ -376,6 +376,7 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
 
 **Other removals**
 
+- Removed the previously deprecated :meth:`Index.summary` (:issue:`18217`)
 - Removed the previously deprecated :meth:`Series.get_value`, :meth:`Series.set_value`, :meth:`DataFrame.get_value`, :meth:`DataFrame.set_value` (:issue:`17739`)
 - Changed the the default value of `inplace` in :meth:`DataFrame.set_index` and :meth:`Series.set_axis`. It now defaults to False (:issue:`27600`)
 - Removed support for nested renaming in :meth:`DataFrame.aggregate`, :meth:`Series.aggregate`, :meth:`DataFrameGroupBy.aggregate`, :meth:`SeriesGroupBy.aggregate`, :meth:`Rolling.aggregate` (:issue:`18529`)
@@ -391,6 +392,7 @@ or ``matplotlib.Axes.plot``. See :ref:`plotting.formatters` for more.
 - Removed the previously deprecated :meth:`ExtensionArray._formatting_values`. Use :attr:`ExtensionArray._formatter` instead. (:issue:`23601`)
 - Removed support for legacy HDF5 formats (:issue:`29787`)
 - :func:`read_excel` removed support for "skip_footer" argument, use "skipfooter" instead (:issue:`18836`)
+- :func:`read_excel` no longer allows an integer value for the parameter ``usecols``, instead pass a list of integers from 0 to ``usecols`` inclusive (:issue:`23635`)
 - :meth:`DataFrame.to_records` no longer supports the argument "convert_datetime64" (:issue:`18902`)
 - Removed the previously deprecated ``IntervalIndex.from_intervals`` in favor of the :class:`IntervalIndex` constructor (:issue:`19263`)
 - Changed the default value for the "keep_tz" argument in :meth:`DatetimeIndex.to_series` to ``True`` (:issue:`23739`)
@@ -595,6 +597,7 @@ Reshaping
 
 - Bug in :meth:`DataFrame.apply` that caused incorrect output with empty :class:`DataFrame` (:issue:`28202`, :issue:`21959`)
 - Bug in :meth:`DataFrame.stack` not handling non-unique indexes correctly when creating MultiIndex (:issue: `28301`)
+- Bug in :meth:`pivot_table` not returning correct type ``float`` when ``margins=True`` and ``aggfunc='mean'`` (:issue:`24893`)
 - Bug :func:`merge_asof` could not use :class:`datetime.timedelta` for ``tolerance`` kwarg (:issue:`28098`)
 - Bug in :func:`merge`, did not append suffixes correctly with MultiIndex (:issue:`28518`)
 - :func:`qcut` and :func:`cut` now handle boolean input (:issue:`20303`)
@@ -603,6 +606,8 @@ Reshaping
 - Bug :meth:`Series.pct_change` where supplying an anchored frequency would throw a ValueError (:issue:`28664`)
 - Bug where :meth:`DataFrame.equals` returned True incorrectly in some cases when two DataFrames had the same columns in different orders (:issue:`28839`)
 - Bug in :meth:`DataFrame.replace` that caused non-numeric replacer's dtype not respected (:issue:`26632`)
+- Bug in :func:`melt` where supplying mixed strings and numeric values for ``id_vars`` or ``value_vars`` would incorrectly raise a ``ValueError`` (:issue:`29718`)
+
 
 Sparse
 ^^^^^^

pandas/_libs/tslibs/strptime.pyx

@@ -110,7 +110,7 @@ def array_strptime(object[:] values, object fmt,
                                  f"in format '{fmt}'")
             # IndexError only occurs when the format string is "%"
             except IndexError:
-                raise ValueError("stray % in format '{fmt}'")
+                raise ValueError(f"stray % in format '{fmt}'")
             _regex_cache[fmt] = format_regex
 
     result = np.empty(n, dtype='M8[ns]')

pandas/_libs/window/__init__.py

@@ -1,5 +1,7 @@
 # cython: boundscheck=False, wraparound=False, cdivision=True
 
+from typing import Tuple
+
 import numpy as np
 from numpy cimport ndarray, int64_t
 
@@ -8,33 +10,6 @@ from numpy cimport ndarray, int64_t
 # These define start/end indexers to compute offsets
 
 
-class MockFixedWindowIndexer:
-    """
-
-    We are just checking parameters of the indexer,
-    and returning a consistent API with fixed/variable
-    indexers.
-
-    Parameters
-    ----------
-    values: ndarray
-        values data array
-    win: int64_t
-        window size
-    index: object
-        index of the values
-    closed: string
-        closed behavior
-    """
-    def __init__(self, ndarray values, int64_t win, object closed, object index=None):
-
-        self.start = np.empty(0, dtype='int64')
-        self.end = np.empty(0, dtype='int64')
-
-    def get_window_bounds(self):
-        return self.start, self.end
-
-
 class FixedWindowIndexer:
     """
     create a fixed length window indexer object
@@ -66,7 +41,7 @@ class FixedWindowIndexer:
         end_e = start_e + win
         self.end = np.concatenate([end_s, end_e])[:N]
 
-    def get_window_bounds(self):
+    def get_window_bounds(self) -> Tuple[np.ndarray, np.ndarray]:
         return self.start, self.end
 
 
@@ -108,7 +83,7 @@ class VariableWindowIndexer:
 
     @staticmethod
     def build(const int64_t[:] index, int64_t win, bint left_closed,
-              bint right_closed, int64_t N):
+              bint right_closed, int64_t N) -> Tuple[np.ndarray, np.ndarray]:
 
         cdef:
             ndarray[int64_t] start, end
@@ -161,5 +136,5 @@ class VariableWindowIndexer:
                     end[i] -= 1
         return start, end
 
-    def get_window_bounds(self):
+    def get_window_bounds(self) -> Tuple[np.ndarray, np.ndarray]:
         return self.start, self.end

pandas/_libs/window.pyx renamed to pandas/_libs/window/aggregations.pyx

@@ -1132,19 +1132,6 @@ def _summary(self, name=None):
             name = type(self).__name__
         return f"{name}: {len(self)} entries{index_summary}"
 
-    def summary(self, name=None):
-        """
-        Return a summarized representation.
-
-        .. deprecated:: 0.23.0
-        """
-        warnings.warn(
-            "'summary' is deprecated and will be removed in a future version.",
-            FutureWarning,
-            stacklevel=2,
-        )
-        return self._summary(name)
-
     # --------------------------------------------------------------------
     # Conversion Methods
 

pandas/_libs/window_indexer.pyx renamed to pandas/_libs/window/indexers.pyx

@@ -11,6 +11,7 @@
 from pandas.core.dtypes.missing import notna
 
 from pandas.core.arrays import Categorical
+import pandas.core.common as com
 from pandas.core.frame import DataFrame, _shared_docs
 from pandas.core.indexes.base import Index
 from pandas.core.reshape.concat import concat
@@ -47,7 +48,7 @@ def melt(
         else:
             # Check that `id_vars` are in frame
             id_vars = list(id_vars)
-            missing = Index(np.ravel(id_vars)).difference(cols)
+            missing = Index(com.flatten(id_vars)).difference(cols)
             if not missing.empty:
                 raise KeyError(
                     "The following 'id_vars' are not present"
@@ -69,7 +70,7 @@ def melt(
         else:
             value_vars = list(value_vars)
             # Check that `value_vars` are in frame
-            missing = Index(np.ravel(value_vars)).difference(cols)
+            missing = Index(com.flatten(value_vars)).difference(cols)
             if not missing.empty:
                 raise KeyError(
                     "The following 'value_vars' are not present in"

pandas/core/indexes/base.py

@@ -261,9 +261,12 @@ def _add_margins(
 
     row_names = result.index.names
     try:
+        # check the result column and leave floats
         for dtype in set(result.dtypes):
             cols = result.select_dtypes([dtype]).columns
-            margin_dummy[cols] = margin_dummy[cols].astype(dtype)
+            margin_dummy[cols] = margin_dummy[cols].apply(
+                maybe_downcast_to_dtype, args=(dtype,)
+            )
         result = result.append(margin_dummy)
     except TypeError:
 

pandas/core/reshape/melt.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 
-import pandas._libs.window as libwindow
+import pandas._libs.window.aggregations as window_aggregations
 from pandas.compat.numpy import function as nv
 from pandas.util._decorators import Appender, Substitution
 
@@ -228,11 +228,11 @@ def _apply(self, func, **kwargs):
 
             # if we have a string function name, wrap it
             if isinstance(func, str):
-                cfunc = getattr(libwindow, func, None)
+                cfunc = getattr(window_aggregations, func, None)
                 if cfunc is None:
                     raise ValueError(
                         "we do not support this function "
-                        "in libwindow.{func}".format(func=func)
+                        "in window_aggregations.{func}".format(func=func)
                     )
 
                 def func(arg):
@@ -284,7 +284,7 @@ def var(self, bias=False, *args, **kwargs):
         nv.validate_window_func("var", args, kwargs)
 
         def f(arg):
-            return libwindow.ewmcov(
+            return window_aggregations.ewmcov(
                 arg,
                 arg,
                 self.com,
@@ -328,7 +328,7 @@ def cov(self, other=None, pairwise=None, bias=False, **kwargs):
         def _get_cov(X, Y):
             X = self._shallow_copy(X)
             Y = self._shallow_copy(Y)
-            cov = libwindow.ewmcov(
+            cov = window_aggregations.ewmcov(
                 X._prep_values(),
                 Y._prep_values(),
                 self.com,
@@ -375,7 +375,7 @@ def _get_corr(X, Y):
             Y = self._shallow_copy(Y)
 
             def _cov(x, y):
-                return libwindow.ewmcov(
+                return window_aggregations.ewmcov(
                     x,
                     y,
                     self.com,

pandas/core/reshape/pivot.py

@@ -10,8 +10,8 @@
 
 import numpy as np
 
-import pandas._libs.window as libwindow
-import pandas._libs.window_indexer as libwindow_indexer
+import pandas._libs.window.aggregations as window_aggregations
+import pandas._libs.window.indexers as window_indexers
 from pandas.compat._optional import import_optional_dependency
 from pandas.compat.numpy import function as nv
 from pandas.util._decorators import Appender, Substitution, cache_readonly
@@ -381,11 +381,11 @@ def _get_roll_func(self, func_name: str) -> Callable:
         -------
         func : callable
         """
-        window_func = getattr(libwindow, func_name, None)
+        window_func = getattr(window_aggregations, func_name, None)
         if window_func is None:
             raise ValueError(
                 "we do not support this function "
-                "in libwindow.{func_name}".format(func_name=func_name)
+                "in window_aggregations.{func_name}".format(func_name=func_name)
             )
         return window_func
 
@@ -406,8 +406,8 @@ def _get_window_indexer(self):
         Return an indexer class that will compute the window start and end bounds
         """
         if self.is_freq_type:
-            return libwindow_indexer.VariableWindowIndexer
-        return libwindow_indexer.FixedWindowIndexer
+            return window_indexers.VariableWindowIndexer
+        return window_indexers.FixedWindowIndexer
 
     def _apply(
         self,

pandas/core/window/ewm.py

@@ -166,8 +166,9 @@
       result 'foo'
 
     If a column or index contains an unparseable date, the entire column or
-    index will be returned unaltered as an object data type. For non-standard
-    datetime parsing, use ``pd.to_datetime`` after ``pd.read_excel``.
+    index will be returned unaltered as an object data type. If you don`t want to
+    parse some cells as date just change their type in Excel to "Text".
+    For non-standard datetime parsing, use ``pd.to_datetime`` after ``pd.read_excel``.
 
     Note: A fast-path exists for iso8601-formatted dates.
 date_parser : function, optional

pandas/core/window/rolling.py

@@ -1,5 +1,3 @@
-import warnings
-
 from pandas.compat._optional import import_optional_dependency
 
 from pandas.core.dtypes.common import is_integer, is_list_like
@@ -136,16 +134,11 @@ def _maybe_convert_usecols(usecols):
         return usecols
 
     if is_integer(usecols):
-        warnings.warn(
-            (
-                "Passing in an integer for `usecols` has been "
-                "deprecated. Please pass in a list of int from "
-                "0 to `usecols` inclusive instead."
-            ),
-            FutureWarning,
-            stacklevel=2,
+        raise ValueError(
+            "Passing an integer for `usecols` is no longer supported.  "
+            "Please pass in a list of int from 0 to `usecols` "
+            "inclusive instead."
         )
-        return list(range(usecols + 1))
 
     if isinstance(usecols, str):
         return _range2cols(usecols)