Merge branch 'master' of https://github.com/pandas-dev/pandas into inline_cleanup

Author: jbrockmendel

appveyor.yml

@@ -74,12 +74,18 @@ install:
   # create our env
   - cmd: conda create -n pandas python=%PYTHON_VERSION% cython pytest>=3.1.0 pytest-xdist
   - cmd: activate pandas
+  - cmd: pip install moto
   - SET REQ=ci\requirements-%PYTHON_VERSION%_WIN.run
   - cmd: echo "installing requirements from %REQ%"
   - cmd: conda install -n pandas --file=%REQ%
   - cmd: conda list -n pandas
   - cmd: echo "installing requirements from %REQ% - done"
 
+  # add some pip only reqs to the env
+  - SET REQ=ci\requirements-%PYTHON_VERSION%_WIN.pip
+  - cmd: echo "installing requirements from %REQ%"
+  - cmd: pip install -Ur %REQ%
+
   # build em using the local source checkout in the correct windows env
   - cmd: '%CMD_IN_ENV% python setup.py build_ext --inplace'
 

asv_bench/benchmarks/categoricals.py

@@ -67,6 +67,9 @@ def time_value_counts_dropna(self):
     def time_rendering(self):
         str(self.sel)
 
+    def time_set_categories(self):
+        self.ts.cat.set_categories(self.ts.cat.categories[::2])
+
 
 class Categoricals3(object):
     goal_time = 0.2

asv_bench/benchmarks/index_object.py

@@ -199,3 +199,23 @@ def time_datetime_level_values_full(self):
 
     def time_datetime_level_values_sliced(self):
         self.mi[:10].values
+
+
+class Range(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.idx_inc = RangeIndex(start=0, stop=10**7, step=3)
+        self.idx_dec = RangeIndex(start=10**7, stop=-1, step=-3)
+
+    def time_max(self):
+        self.idx_inc.max()
+
+    def time_max_trivial(self):
+        self.idx_dec.max()
+
+    def time_min(self):
+        self.idx_dec.min()
+
+    def time_min_trivial(self):
+        self.idx_inc.min()

asv_bench/benchmarks/period.py

@@ -78,6 +78,65 @@ def time_value_counts_pindex(self):
         self.i.value_counts()
 
 
+class Properties(object):
+    def setup(self):
+        self.per = Period('2017-09-06 08:28', freq='min')
+
+    def time_year(self):
+        self.per.year
+
+    def time_month(self):
+        self.per.month
+
+    def time_day(self):
+        self.per.day
+
+    def time_hour(self):
+        self.per.hour
+
+    def time_minute(self):
+        self.per.minute
+
+    def time_second(self):
+        self.per.second
+
+    def time_is_leap_year(self):
+        self.per.is_leap_year
+
+    def time_quarter(self):
+        self.per.quarter
+
+    def time_qyear(self):
+        self.per.qyear
+
+    def time_week(self):
+        self.per.week
+
+    def time_daysinmonth(self):
+        self.per.daysinmonth
+
+    def time_dayofweek(self):
+        self.per.dayofweek
+
+    def time_dayofyear(self):
+        self.per.dayofyear
+
+    def time_start_time(self):
+        self.per.start_time
+
+    def time_end_time(self):
+        self.per.end_time
+
+    def time_to_timestamp():
+        self.per.to_timestamp()
+
+    def time_now():
+        self.per.now()
+
+    def time_asfreq():
+        self.per.asfreq('A')
+
+
 class period_standard_indexing(object):
     goal_time = 0.2
 

asv_bench/benchmarks/timestamp.py

@@ -1,5 +1,7 @@
 from .pandas_vb_common import *
 from pandas import to_timedelta, Timestamp
+import pytz
+import datetime
 
 
 class TimestampProperties(object):
@@ -58,3 +60,30 @@ def time_is_leap_year(self):
 
     def time_microsecond(self):
         self.ts.microsecond
+
+
+class TimestampOps(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.ts = Timestamp('2017-08-25 08:16:14')
+        self.ts_tz = Timestamp('2017-08-25 08:16:14', tz='US/Eastern')
+
+        dt = datetime.datetime(2016, 3, 27, 1)
+        self.tzinfo = pytz.timezone('CET').localize(dt, is_dst=False).tzinfo
+        self.ts2 = Timestamp(dt)
+
+    def time_replace_tz(self):
+        self.ts.replace(tzinfo=pytz.timezone('US/Eastern'))
+
+    def time_replace_across_dst(self):
+        self.ts2.replace(tzinfo=self.tzinfo)
+
+    def time_replace_None(self):
+        self.ts_tz.replace(tzinfo=None)
+
+    def time_to_pydatetime(self):
+        self.ts.to_pydatetime()
+
+    def time_to_pydatetime_tz(self):
+        self.ts_tz.to_pydatetime()

ci/install_circle.sh

@@ -67,6 +67,7 @@ time conda create -n pandas -q --file=${REQ_BUILD} || exit 1
 time conda install -n pandas pytest>=3.1.0 || exit 1
 
 source activate pandas
+time pip install moto || exit 1
 
 # build but don't install
 echo "[build em]"

ci/install_travis.sh

@@ -104,7 +104,7 @@ if [ -e ${REQ} ]; then
 fi
 
 time conda install -n pandas pytest>=3.1.0
-time pip install pytest-xdist
+time pip install pytest-xdist moto
 
 if [ "$LINT" ]; then
    conda install flake8

ci/requirements-2.7_WIN.pip

@@ -8,7 +8,6 @@ xlrd
 xlwt
 scipy
 feather-format
-pyarrow
 numexpr
 pytables
 matplotlib

ci/requirements-3.6_NUMPY_DEV.pip

@@ -5,3 +5,4 @@ cython
 pytest>=3.1.0
 pytest-cov
 flake8
+moto

ci/requirements-3.6_WIN.pip

@@ -3,9 +3,11 @@
 Contributing to the documentation
 =================================
 
-If you're not the developer type, contributing to the documentation is still
-of huge value. You don't even have to be an expert on
-*pandas* to do so! Something as simple as rewriting small passages for clarity
+Whether you are someone who loves writing, teaching, or development,
+contributing to the documentation is a huge value. If you don't see yourself
+as a developer type, please don't stress and know that we want you to
+contribute. You don't even have to be an expert on *pandas* to do so!
+Something as simple as rewriting small passages for clarity
 as you reference the docs is a simple but effective way to contribute. The
 next person to read that passage will be in your debt!
 

ci/requirements-3.6_WIN.run

@@ -625,7 +625,7 @@ Index Types
 We have discussed ``MultiIndex`` in the previous sections pretty extensively. ``DatetimeIndex`` and ``PeriodIndex``
 are shown :ref:`here <timeseries.overview>`. ``TimedeltaIndex`` are :ref:`here <timedeltas.timedeltas>`.
 
-In the following sub-sections we will highlite some other index types.
+In the following sub-sections we will highlight some other index types.
 
 .. _indexing.categoricalindex:
 
@@ -645,7 +645,7 @@ and allows efficient indexing and storage of an index with a large number of dup
    df.dtypes
    df.B.cat.categories
 
-Setting the index, will create create a ``CategoricalIndex``
+Setting the index, will create a ``CategoricalIndex``
 
 .. ipython:: python
 
@@ -681,7 +681,7 @@ Groupby operations on the index will preserve the index nature as well
 Reindexing operations, will return a resulting index based on the type of the passed
 indexer, meaning that passing a list will return a plain-old-``Index``; indexing with
 a ``Categorical`` will return a ``CategoricalIndex``, indexed according to the categories
-of the PASSED ``Categorical`` dtype. This allows one to arbitrarly index these even with
+of the PASSED ``Categorical`` dtype. This allows one to arbitrarily index these even with
 values NOT in the categories, similarly to how you can reindex ANY pandas index.
 
 .. ipython :: python
@@ -722,7 +722,7 @@ Int64Index and RangeIndex
 Prior to 0.18.0, the ``Int64Index`` would provide the default index for all ``NDFrame`` objects.
 
 ``RangeIndex`` is a sub-class of ``Int64Index`` added in version 0.18.0, now providing the default index for all ``NDFrame`` objects.
-``RangeIndex`` is an optimized version of ``Int64Index`` that can represent a monotonic ordered set. These are analagous to python `range types <https://docs.python.org/3/library/stdtypes.html#typesseq-range>`__.
+``RangeIndex`` is an optimized version of ``Int64Index`` that can represent a monotonic ordered set. These are analogous to python `range types <https://docs.python.org/3/library/stdtypes.html#typesseq-range>`__.
 
 .. _indexing.float64index:
 
@@ -963,7 +963,7 @@ index can be somewhat complicated. For example, the following does not work:
     s.loc['c':'e'+1]
 
 A very common use case is to limit a time series to start and end at two
-specific dates. To enable this, we made the design design to make label-based
+specific dates. To enable this, we made the design to make label-based
 slicing include both endpoints:
 
 .. ipython:: python

ci/requirements_dev.txt

@@ -218,10 +218,19 @@ Top-level dealing with datetimelike
    to_timedelta
    date_range
    bdate_range
+   cdate_range
    period_range
    timedelta_range
    infer_freq
 
+Top-level dealing with intervals
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   interval_range
+
 Top-level evaluation
 ~~~~~~~~~~~~~~~~~~~~
 
@@ -1282,7 +1291,7 @@ Index
 -----
 
 **Many of these methods or variants thereof are available on the objects
-that contain an index (Series/Dataframe) and those should most likely be
+that contain an index (Series/DataFrame) and those should most likely be
 used before calling these methods directly.**
 
 .. autosummary::
@@ -1407,6 +1416,20 @@ Selecting
    Index.slice_indexer
    Index.slice_locs
 
+.. _api.numericindex:
+
+Numeric Index
+-------------
+
+.. autosummary::
+   :toctree: generated/
+   :template: autosummary/class_without_autosummary.rst
+
+   RangeIndex
+   Int64Index
+   UInt64Index
+   Float64Index
+
 .. _api.categoricalindex:
 
 CategoricalIndex
@@ -2016,6 +2039,7 @@ Upsampling
    Resampler.backfill
    Resampler.bfill
    Resampler.pad
+   Resampler.nearest
    Resampler.fillna
    Resampler.asfreq
    Resampler.interpolate

doc/README.rst

@@ -923,7 +923,7 @@ Passing a named function will yield that name for the row:
 Aggregating with a dict
 +++++++++++++++++++++++
 
-Passing a dictionary of column names to a scalar or a list of scalars, to ``DataFame.agg``
+Passing a dictionary of column names to a scalar or a list of scalars, to ``DataFrame.agg``
 allows you to customize which functions are applied to which columns. Note that the results
 are not in any particular order, you can use an ``OrderedDict`` instead to guarantee ordering.
 

doc/source/advanced.rst

@@ -146,6 +146,8 @@ Using ``.describe()`` on categorical data will produce similar output to a `Seri
     df.describe()
     df["cat"].describe()
 
+.. _categorical.cat:
+
 Working with categories
 -----------------------
 
@@ -204,6 +206,10 @@ by using the :func:`Categorical.rename_categories` method:
     s.cat.categories = ["Group %s" % g for g in s.cat.categories]
     s
     s.cat.rename_categories([1,2,3])
+    s
+    # You can also pass a dict-like object to map the renaming
+    s.cat.rename_categories({1: 'x', 2: 'y', 3: 'z'})
+    s
 
 .. note::
 

doc/source/api.rst

@@ -654,7 +654,7 @@ aggregation with, outputting a DataFrame:
 
    r['A'].agg([np.sum, np.mean, np.std])
 
-On a widowed DataFrame, you can pass a list of functions to apply to each
+On a windowed DataFrame, you can pass a list of functions to apply to each
 column, which produces an aggregated result with a hierarchical index:
 
 .. ipython:: python

doc/source/basics.rst

@@ -561,7 +561,7 @@ must be either implemented on GroupBy or available via :ref:`dispatching
 
 .. note::
 
-    If you pass a dict to ``aggregate``, the ordering of the output colums is
+    If you pass a dict to ``aggregate``, the ordering of the output columns is
     non-deterministic. If you want to be sure the output columns will be in a specific
     order, you can use an ``OrderedDict``.  Compare the output of the following two commands:
 
@@ -1211,7 +1211,7 @@ Groupby by Indexer to 'resample' data
 
 Resampling produces new hypothetical samples (resamples) from already existing observed data or from a model that generates data. These new samples are similar to the pre-existing samples.
 
-In order to resample to work on indices that are non-datetimelike , the following procedure can be utilized.
+In order to resample to work on indices that are non-datetimelike, the following procedure can be utilized.
 
 In the following examples, **df.index // 5** returns a binary array which is used to determine what gets selected for the groupby operation.
 

doc/source/categorical.rst

@@ -714,7 +714,7 @@ Finally, one can also set a seed for ``sample``'s random number generator using
 Setting With Enlargement
 ------------------------
 
-The ``.loc/[]`` operations can perform enlargement when setting a non-existant key for that axis.
+The ``.loc/[]`` operations can perform enlargement when setting a non-existent key for that axis.
 
 In the ``Series`` case this is effectively an appending operation
 

doc/source/computation.rst

@@ -3077,7 +3077,7 @@ Compressed pickle files
 
 .. versionadded:: 0.20.0
 
-:func:`read_pickle`, :meth:`DataFame.to_pickle` and :meth:`Series.to_pickle` can read
+:func:`read_pickle`, :meth:`DataFrame.to_pickle` and :meth:`Series.to_pickle` can read
 and write compressed pickle files. The compression types of ``gzip``, ``bz2``, ``xz`` are supported for reading and writing.
 `zip`` file supports read only and must contain only one data file
 to be read in.
@@ -4492,7 +4492,7 @@ Several caveats.
 - The format will NOT write an ``Index``, or ``MultiIndex`` for the ``DataFrame`` and will raise an
   error if a non-default one is provided. You can simply ``.reset_index(drop=True)`` in order to store the index.
 - Duplicate column names and non-string columns names are not supported
-- Categorical dtypes are currently not-supported (for ``pyarrow``).
+- Categorical dtypes can be serialized to parquet, but will de-serialize as ``object`` dtype.
 - Non supported types include ``Period`` and actual python object types. These will raise a helpful error message
   on an attempt at serialization.