Merge branch 'master' into blacklist-dtypes

Author: gwrome

doc/source/whatsnew/v0.25.0.rst

@@ -361,6 +361,7 @@ I/O
 - Bug in ``read_csv`` which would not raise ``ValueError`` if a column index in ``usecols`` was out of bounds (:issue:`25623`)
 - Improved the explanation for the failure when value labels are repeated in Stata dta files and suggested work-arounds (:issue:`25772`)
 - Improved :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` to read incorrectly formatted 118 format files saved by Stata (:issue:`25960`)
+- Fixed bug in loading objects from S3 that contain ``#`` characters in the URL (:issue:`25945`)
 
 Plotting
 ^^^^^^^^

mypy.ini

@@ -158,9 +158,6 @@ ignore_errors=True
 [mypy-pandas.io.pytables]
 ignore_errors=True
 
-[mypy-pandas.io.stata]
-ignore_errors=True
-
 [mypy-pandas.util._doctools]
 ignore_errors=True
 

pandas/__init__.py

@@ -20,7 +20,9 @@
 del hard_dependencies, dependency, missing_dependencies
 
 # numpy compat
-from pandas.compat.numpy import *
+from pandas.compat.numpy import (
+    _np_version_under1p14, _np_version_under1p15, _np_version_under1p16,
+    _np_version_under1p17)
 
 try:
     from pandas._libs import (hashtable as _hashtable,
@@ -42,14 +44,72 @@
 # let init-time option registration happen
 import pandas.core.config_init
 
-from pandas.core.api import *
-from pandas.core.sparse.api import *
-from pandas.tseries.api import *
-from pandas.core.computation.api import *
-from pandas.core.reshape.api import *
+from pandas.core.api import (
+    # dtype
+    Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype, UInt8Dtype,
+    UInt16Dtype, UInt32Dtype, UInt64Dtype, CategoricalDtype,
+    PeriodDtype, IntervalDtype, DatetimeTZDtype,
+
+    # missing
+    isna, isnull, notna, notnull,
+
+    # indexes
+    Index, CategoricalIndex, Int64Index, UInt64Index, RangeIndex,
+    Float64Index, MultiIndex, IntervalIndex, TimedeltaIndex,
+    DatetimeIndex, PeriodIndex, IndexSlice,
+
+    # tseries
+    NaT, Period, period_range, Timedelta, timedelta_range,
+    Timestamp, date_range, bdate_range, Interval, interval_range,
+    DateOffset,
+
+    # conversion
+    to_numeric, to_datetime, to_timedelta,
+
+    # misc
+    np, TimeGrouper, Grouper, factorize, unique, value_counts,
+    array, Categorical, set_eng_float_format, Series, DataFrame,
+    Panel)
+
+from pandas.core.sparse.api import (
+    SparseArray, SparseDataFrame, SparseSeries, SparseDtype)
+
+from pandas.tseries.api import infer_freq
+from pandas.tseries import offsets
+
+from pandas.core.computation.api import eval
+
+from pandas.core.reshape.api import (
+    concat, lreshape, melt, wide_to_long, merge, merge_asof,
+    merge_ordered, crosstab, pivot, pivot_table, get_dummies,
+    cut, qcut)
 
 from pandas.util._print_versions import show_versions
-from pandas.io.api import *
+
+from pandas.io.api import (
+    # excel
+    ExcelFile, ExcelWriter, read_excel,
+
+    # packers
+    read_msgpack, to_msgpack,
+
+    # parsers
+    read_csv, read_fwf, read_table,
+
+    # pickle
+    read_pickle, to_pickle,
+
+    # pytables
+    HDFStore, read_hdf,
+
+    # sql
+    read_sql, read_sql_query,
+    read_sql_table,
+
+    # misc
+    read_clipboard, read_parquet, read_feather, read_gbq,
+    read_html, read_json, read_stata, read_sas)
+
 from pandas.util._tester import test
 import pandas.testing
 import pandas.arrays

pandas/_typing.py

@@ -1,4 +1,11 @@
 from pathlib import Path
 from typing import IO, AnyStr, Union
 
+import numpy as np
+
+from pandas.core.dtypes.dtypes import ExtensionDtype
+from pandas.core.dtypes.generic import ABCExtensionArray
+
+ArrayLike = Union[ABCExtensionArray, np.ndarray]
+Dtype = Union[str, np.dtype, ExtensionDtype]
 FilePathOrBuffer = Union[str, Path, IO[AnyStr]]

pandas/compat/__init__.py

@@ -8,7 +8,6 @@
 * lists: lrange(), lmap(), lzip(), lfilter()
 * iterable method compatibility: iteritems, iterkeys, itervalues
   * Uses the original method if available, otherwise uses items, keys, values.
-* bind_method: binds functions to classes
 * add_metaclass(metaclass) - class decorator that recreates class with with the
   given metaclass instead (and avoids intermediary class creation)
 
@@ -22,18 +21,14 @@
 from distutils.version import LooseVersion
 import sys
 import platform
-import types
 import struct
 
 PY2 = sys.version_info[0] == 2
 PY3 = sys.version_info[0] >= 3
-PY35 = sys.version_info >= (3, 5)
 PY36 = sys.version_info >= (3, 6)
 PY37 = sys.version_info >= (3, 7)
 PYPY = platform.python_implementation() == 'PyPy'
 
-from pandas.compat.chainmap import DeepChainMap
-
 
 # list-producing versions of the major Python iterating functions
 def lrange(*args, **kwargs):
@@ -52,29 +47,6 @@ def lfilter(*args, **kwargs):
     return list(filter(*args, **kwargs))
 
 
-if PY3:
-    def isidentifier(s):
-        return s.isidentifier()
-
-    def str_to_bytes(s, encoding=None):
-        return s.encode(encoding or 'ascii')
-
-    def bytes_to_str(b, encoding=None):
-        return b.decode(encoding or 'utf-8')
-
-else:
-    # Python 2
-    _name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*$")
-
-    def isidentifier(s, dotted=False):
-        return bool(_name_re.match(s))
-
-    def str_to_bytes(s, encoding='ascii'):
-        return s
-
-    def bytes_to_str(b, encoding='ascii'):
-        return b
-
 if PY2:
     def iteritems(obj, **kw):
         return obj.iteritems(**kw)
@@ -95,30 +67,6 @@ def iterkeys(obj, **kw):
     def itervalues(obj, **kw):
         return iter(obj.values(**kw))
 
-
-def bind_method(cls, name, func):
-    """Bind a method to class, python 2 and python 3 compatible.
-
-    Parameters
-    ----------
-
-    cls : type
-        class to receive bound method
-    name : basestring
-        name of method on class instance
-    func : function
-        function to be bound as method
-
-
-    Returns
-    -------
-    None
-    """
-    # only python 2 has bound/unbound method issue
-    if not PY3:
-        setattr(cls, name, types.MethodType(func, None, cls))
-    else:
-        setattr(cls, name, func)
 # ----------------------------------------------------------------------------
 # functions largely based / taken from the six module
 
@@ -133,7 +81,7 @@ def to_str(s):
         Convert bytes and non-string into Python 3 str
         """
         if isinstance(s, bytes):
-            s = bytes_to_str(s)
+            s = s.decode('utf-8')
         elif not isinstance(s, str):
             s = str(s)
         return s
@@ -172,6 +120,7 @@ def wrapper(cls):
         return metaclass(cls.__name__, cls.__bases__, orig_vars)
     return wrapper
 
+
 if PY3:
     def raise_with_traceback(exc, traceback=Ellipsis):
         if traceback == Ellipsis:
@@ -207,6 +156,7 @@ def raise_with_traceback(exc, traceback=Ellipsis):
 else:
     re_type = type(re.compile(''))
 
+
 # https://github.com/pandas-dev/pandas/pull/9123
 def is_platform_little_endian():
     """ am I little endian """

pandas/core/arrays/base.py

@@ -21,6 +21,7 @@
     ABCExtensionArray, ABCIndexClass, ABCSeries)
 from pandas.core.dtypes.missing import isna
 
+from pandas._typing import ArrayLike
 from pandas.core import ops
 
 _not_implemented_message = "{} does not implement {}."
@@ -338,7 +339,7 @@ def astype(self, dtype, copy=True):
         """
         return np.array(self, dtype=dtype, copy=copy)
 
-    def isna(self) -> Union[ABCExtensionArray, np.ndarray]:
+    def isna(self) -> ArrayLike:
         """
         A 1-D array indicating if each value is missing.
 

pandas/core/arrays/sparse.py

@@ -5,7 +5,7 @@
 import numbers
 import operator
 import re
-from typing import Any, Callable, Type, Union
+from typing import Any, Callable
 import warnings
 
 import numpy as np
@@ -31,6 +31,7 @@
     ABCIndexClass, ABCSeries, ABCSparseArray, ABCSparseSeries)
 from pandas.core.dtypes.missing import isna, na_value_for_dtype, notna
 
+from pandas._typing import Dtype
 from pandas.core.accessor import PandasDelegate, delegate_names
 import pandas.core.algorithms as algos
 from pandas.core.arrays import ExtensionArray, ExtensionOpsMixin
@@ -80,7 +81,7 @@ class SparseDtype(ExtensionDtype):
 
     def __init__(
             self,
-            dtype: Union[str, np.dtype, ExtensionDtype, Type] = np.float64,
+            dtype: Dtype = np.float64,
             fill_value: Any = None
     ) -> None:
         from pandas.core.dtypes.missing import na_value_for_dtype

pandas/core/computation/expr.py

@@ -10,10 +10,9 @@
 
 import numpy as np
 
-from pandas.compat import lmap
+from pandas.compat import iteritems, lmap
 
 import pandas as pd
-from pandas import compat
 from pandas.core import common as com
 from pandas.core.base import StringMixin
 from pandas.core.computation.common import (
@@ -301,7 +300,7 @@ def f(self, node, *args, **kwargs):
 def add_ops(op_classes):
     """Decorator to add default implementation of ops."""
     def f(cls):
-        for op_attr_name, op_class in compat.iteritems(op_classes):
+        for op_attr_name, op_class in iteritems(op_classes):
             ops = getattr(cls, '{name}_ops'.format(name=op_attr_name))
             ops_map = getattr(cls, '{name}_op_nodes_map'.format(
                 name=op_attr_name))
@@ -590,9 +589,7 @@ def visit_Attribute(self, node, **kwargs):
         raise ValueError("Invalid Attribute context {name}"
                          .format(name=ctx.__name__))
 
-    def visit_Call_35(self, node, side=None, **kwargs):
-        """ in 3.5 the starargs attribute was changed to be more flexible,
-        #11097 """
+    def visit_Call(self, node, side=None, **kwargs):
 
         if isinstance(node.func, ast.Attribute):
             res = self.visit_Attribute(node.func)
@@ -641,58 +638,6 @@ def visit_Call_35(self, node, side=None, **kwargs):
 
             return self.const_type(res(*new_args, **kwargs), self.env)
 
-    def visit_Call_legacy(self, node, side=None, **kwargs):
-
-        # this can happen with: datetime.datetime
-        if isinstance(node.func, ast.Attribute):
-            res = self.visit_Attribute(node.func)
-        elif not isinstance(node.func, ast.Name):
-            raise TypeError("Only named functions are supported")
-        else:
-            try:
-                res = self.visit(node.func)
-            except UndefinedVariableError:
-                # Check if this is a supported function name
-                try:
-                    res = FuncNode(node.func.id)
-                except ValueError:
-                    # Raise original error
-                    raise
-
-        if res is None:
-            raise ValueError("Invalid function call {func}"
-                             .format(func=node.func.id))
-        if hasattr(res, 'value'):
-            res = res.value
-
-        if isinstance(res, FuncNode):
-            args = [self.visit(targ) for targ in node.args]
-
-            if node.starargs is not None:
-                args += self.visit(node.starargs)
-
-            if node.keywords or node.kwargs:
-                raise TypeError("Function \"{name}\" does not support keyword "
-                                "arguments".format(name=res.name))
-
-            return res(*args, **kwargs)
-
-        else:
-            args = [self.visit(targ).value for targ in node.args]
-            if node.starargs is not None:
-                args += self.visit(node.starargs).value
-
-            keywords = {}
-            for key in node.keywords:
-                if not isinstance(key, ast.keyword):
-                    raise ValueError("keyword error in function call "
-                                     "'{func}'".format(func=node.func.id))
-                keywords[key.arg] = self.visit(key.value).value
-            if node.kwargs is not None:
-                keywords.update(self.visit(node.kwargs).value)
-
-            return self.const_type(res(*args, **keywords), self.env)
-
     def translate_In(self, op):
         return op
 
@@ -734,14 +679,6 @@ def visitor(x, y):
         return reduce(visitor, operands)
 
 
-# ast.Call signature changed on 3.5,
-# conditionally change which methods is named
-# visit_Call depending on Python version, #11097
-if compat.PY35:
-    BaseExprVisitor.visit_Call = BaseExprVisitor.visit_Call_35
-else:
-    BaseExprVisitor.visit_Call = BaseExprVisitor.visit_Call_legacy
-
 _python_not_supported = frozenset(['Dict', 'BoolOp', 'In', 'NotIn'])
 _numexpr_supported_calls = frozenset(_reductions + _mathops)
 

pandas/core/computation/pytables.py

@@ -6,7 +6,7 @@
 import numpy as np
 
 from pandas._libs.tslibs import Timedelta, Timestamp
-from pandas.compat import DeepChainMap
+from pandas.compat.chainmap import DeepChainMap
 
 from pandas.core.dtypes.common import is_list_like
 

pandas/core/computation/scope.py

@@ -13,7 +13,7 @@
 import numpy as np
 
 from pandas._libs.tslibs import Timestamp
-from pandas.compat import DeepChainMap
+from pandas.compat.chainmap import DeepChainMap
 
 from pandas.core.base import StringMixin
 import pandas.core.computation as compu