Merge remote-tracking branch 'upstream/master' into pr/18629

Author: jorisvandenbossche

asv_bench/benchmarks/timedelta.py

@@ -1,5 +1,7 @@
-from .pandas_vb_common import *
-from pandas import to_timedelta, Timestamp
+import numpy as np
+import pandas as pd
+
+from pandas import to_timedelta, Timestamp, Timedelta
 
 
 class ToTimedelta(object):
@@ -67,8 +69,8 @@ class DatetimeAccessor(object):
     def setup(self):
         self.N = 100000
         self.series = pd.Series(
-            pd.timedelta_range('1 days', periods=self.N, freq='h')
-        )
+            pd.timedelta_range('1 days', periods=self.N, freq='h'))
+
     def time_dt_accessor(self):
         self.series.dt
 

asv_bench/benchmarks/timestamp.py

@@ -1,8 +1,27 @@
-from pandas import to_timedelta, Timestamp
+from pandas import Timestamp
 import pytz
 import datetime
 
 
+class TimestampConstruction(object):
+    # TODO: classmethod constructors: fromordinal, fromtimestamp...
+
+    def time_parse_iso8601_no_tz(self):
+        Timestamp('2017-08-25 08:16:14')
+
+    def time_parse_iso8601_tz(self):
+        Timestamp('2017-08-25 08:16:14-0500')
+
+    def time_parse_dateutil(self):
+        Timestamp('2017/08/25 08:16:14 AM')
+
+    def time_parse_today(self):
+        Timestamp('today')
+
+    def time_parse_now(self):
+        Timestamp('now')
+
+
 class TimestampProperties(object):
     goal_time = 0.2
 

ci/lint.sh

@@ -24,7 +24,7 @@ if [ "$LINT" ]; then
     echo "Linting setup.py DONE"
 
     echo "Linting asv_bench/benchmarks/"
-    flake8 asv_bench/benchmarks/  --exclude=asv_bench/benchmarks/[ghijoprst]*.py --ignore=F811
+    flake8 asv_bench/benchmarks/  --exclude=asv_bench/benchmarks/[ghijoprs]*.py --ignore=F811
     if [ $? -ne "0" ]; then
         RET=1
     fi

doc/source/io.rst

@@ -4519,6 +4519,7 @@ See the documentation for `pyarrow <http://arrow.apache.org/docs/python/>`__ and
 .. note::
 
    These engines are very similar and should read/write nearly identical parquet format files.
+   Currently ``pyarrow`` does not support timedelta data, and ``fastparquet`` does not support timezone aware datetimes (they are coerced to UTC).
    These libraries differ by having different underlying dependencies (``fastparquet`` by using ``numba``, while ``pyarrow`` uses a c-library).
 
 .. ipython:: python
@@ -4545,8 +4546,8 @@ Read from a parquet file.
 
 .. ipython:: python
 
-   result = pd.read_parquet('example_pa.parquet', engine='pyarrow')
    result = pd.read_parquet('example_fp.parquet', engine='fastparquet')
+   result = pd.read_parquet('example_pa.parquet', engine='pyarrow')
 
    result.dtypes
 

doc/source/whatsnew/v0.21.1.txt

@@ -3,9 +3,23 @@
 v0.21.1
 -------
 
-This is a minor release from 0.21.1 and includes a number of deprecations, new
-features, enhancements, and performance improvements along with a large number
-of bug fixes. We recommend that all users upgrade to this version.
+This is a minor bug-fix release in the 0.21.x series and includes some small regression fixes,
+bug fixes and performance improvements.
+We recommend that all users upgrade to this version.
+
+Highlights include:
+
+- Temporarily restore matplotlib datetime plotting functionality. This should
+  resolve issues for users who relied implicitly on pandas to plot datetimes
+  with matplotlib. See :ref:`here <whatsnew_0211.special>`.
+- Improvements to the Parquet IO functions introduced in 0.21.0. See
+  :ref:`here <whatsnew_0211.enhancements.parquet>`.
+
+
+.. contents:: What's new in v0.21.1
+    :local:
+    :backlinks: none
+
 
 .. _whatsnew_0211.special:
 
@@ -42,9 +56,13 @@ registering them when they want them.
 New features
 ~~~~~~~~~~~~
 
--
--
--
+.. _whatsnew_0211.enhancements.parquet:
+
+Improvements to the Parquet IO functionality
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+- :func:`read_parquet` now allows to specify the columns to read from a parquet file (:issue:`18154`)
+- :func:`read_parquet` now allows to specify kwargs which are passed to the respective engine (:issue:`18216`)
 
 .. _whatsnew_0211.enhancements.other:
 
@@ -54,7 +72,6 @@ Other Enhancements
 - :meth:`Timestamp.timestamp` is now available in Python 2.7. (:issue:`17329`)
 - :class:`Grouper` and :class:`TimeGrouper` now have a friendly repr output (:issue:`18203`).
 - Enabled the use of non-default indexes in :func:`DataFrame.to_parquet` where the underlying engine supports it (:issue:`18581`)
--
 
 .. _whatsnew_0211.deprecations:
 
@@ -70,24 +87,11 @@ Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Improved performance of plotting large series/dataframes (:issue:`18236`).
--
--
-
-.. _whatsnew_0211.docs:
-
-Documentation Changes
-~~~~~~~~~~~~~~~~~~~~~
-
--
--
--
 
 .. _whatsnew_0211.bug_fixes:
 
 Bug Fixes
 ~~~~~~~~~
--
-
 
 Conversion
 ^^^^^^^^^^
@@ -98,7 +102,6 @@ Conversion
 - Bug in :func:`DataFrame.to_dict` where columns of datetime that are tz-aware were not converted to required arrays when used with ``orient='records'``, raising``TypeError` (:issue:`18372`)
 - Bug in :class:`DateTimeIndex` and :meth:`date_range` where mismatching tz-aware ``start`` and ``end`` timezones would not raise an err if ``end.tzinfo`` is None (:issue:`18431`)
 - Bug in :meth:`Series.fillna` which raised when passed a long integer on Python 2 (:issue:`18159`).
--
 
 Indexing
 ^^^^^^^^
@@ -108,7 +111,6 @@ Indexing
 - Bug in :class:`IntervalIndex` constructor when a list of intervals is passed with non-default ``closed`` (:issue:`18334`)
 - Bug in ``Index.putmask`` when an invalid mask passed (:issue:`18368`)
 - Bug in masked assignment of a ``timedelta64[ns]`` dtype ``Series``, incorrectly coerced to float (:issue:`18493`)
--
 
 I/O
 ^^^
@@ -118,8 +120,6 @@ I/O
 - Bug in :func:`read_csv` for handling null values in index columns when specifying ``na_filter=False`` (:issue:`5239`)
 - Bug in :func:`read_csv` when reading numeric category fields with high cardinality (:issue:`18186`)
 - Bug in :meth:`DataFrame.to_csv` when the table had ``MultiIndex`` columns, and a list of strings was passed in for ``header`` (:issue:`5539`)
-- :func:`read_parquet` now allows to specify the columns to read from a parquet file (:issue:`18154`)
-- :func:`read_parquet` now allows to specify kwargs which are passed to the respective engine (:issue:`18216`)
 - Bug in parsing integer datetime-like columns with specified format in ``read_sql`` (:issue:`17855`).
 - Bug in :meth:`DataFrame.to_msgpack` when serializing data of the numpy.bool_ datatype (:issue:`18390`)
 - Bug in :func:`read_json` not decoding when reading line deliminted JSON from S3 (:issue:`17200`)
@@ -130,8 +130,6 @@ Plotting
 ^^^^^^^^
 
 - Bug in ``DataFrame.plot()`` and ``Series.plot()`` with :class:`DatetimeIndex` where a figure generated by them is not pickleable in Python 3 (:issue:`18439`)
--
--
 
 Groupby/Resample/Rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -140,15 +138,6 @@ Groupby/Resample/Rolling
 - Bug in ``DataFrame.resample(...)`` when there is a time change (DST) and resampling frequecy is 12h or higher (:issue:`15549`)
 - Bug in ``pd.DataFrameGroupBy.count()`` when counting over a datetimelike column (:issue:`13393`)
 - Bug in ``rolling.var`` where calculation is inaccurate with a zero-valued array (:issue:`18430`)
--
--
-
-Sparse
-^^^^^^
-
--
--
--
 
 Reshaping
 ^^^^^^^^^
@@ -162,9 +151,6 @@ Numeric
 ^^^^^^^
 
 - Bug in ``pd.Series.rolling.skew()`` and ``rolling.kurt()`` with all equal values has floating issue (:issue:`18044`)
--
--
--
 
 Categorical
 ^^^^^^^^^^^
@@ -179,9 +165,3 @@ String
 ^^^^^^
 
 - :meth:`Series.str.split()` will now propogate ``NaN`` values across all expanded columns instead of ``None`` (:issue:`18450`)
-
-Other
-^^^^^
-
--
--

doc/source/whatsnew/v0.22.0.txt

@@ -135,7 +135,8 @@ Other Enhancements
 - Improved wording of ``ValueError`` raised in :func:`read_csv` when the ``usecols`` argument cannot match all columns. (:issue:`17301`)
 - :func:`DataFrame.corrwith` now silently drops non-numeric columns when passed a Series. Before, an exception was raised (:issue:`18570`).
 - :class:`IntervalIndex` now supports time zone aware ``Interval`` objects (:issue:`18537`, :issue:`18538`)
-
+- :func:`read_excel()` has gained the ``nrows`` parameter (:issue:`16645`)
+- :func:``DataFrame.to_json`` and ``Series.to_json`` now accept an ``index`` argument which allows the user to exclude the index from the JSON output (:issue:`17394`)
 
 .. _whatsnew_0220.api_breaking:
 
@@ -188,6 +189,8 @@ Other API Changes
 - :func:`pandas.DataFrame.merge` no longer casts a ``float`` column to ``object`` when merging on ``int`` and ``float`` columns (:issue:`16572`)
 - The default NA value for :class:`UInt64Index` has changed from 0 to ``NaN``, which impacts methods that mask with NA, such as ``UInt64Index.where()`` (:issue:`18398`)
 - Refactored ``setup.py`` to use ``find_packages`` instead of explicitly listing out all subpackages (:issue:`18535`)
+- Rearranged the order of keyword arguments in :func:`read_excel()` to align with :func:`read_csv()` (:pr:`16672`)
+- :func:`pandas.merge` now raises a ``ValueError`` when trying to merge on incompatible data types (:issue:`9780`)
 
 .. _whatsnew_0220.deprecations:
 
@@ -282,7 +285,7 @@ I/O
 Plotting
 ^^^^^^^^
 
--
+- :func: `DataFrame.plot` now raises a ``ValueError`` when the ``x`` or ``y`` argument is improperly formed (:issue:`18671`)
 -
 -
 

pandas/_libs/src/datetime/np_datetime.c

@@ -527,7 +527,7 @@ void pandas_datetime_to_datetimestruct(npy_datetime val, PANDAS_DATETIMEUNIT fr,
 void pandas_timedelta_to_timedeltastruct(npy_timedelta val,
                                          PANDAS_DATETIMEUNIT fr,
                                          pandas_timedeltastruct *result) {
-  convert_timedelta_to_timedeltastruct(fr, val, result);
+    convert_timedelta_to_timedeltastruct(fr, val, result);
 }
 
 

pandas/_libs/src/datetime/np_datetime_strings.c

@@ -33,55 +33,6 @@ This file implements string parsing and creation for NumPy datetime.
 #include "np_datetime_strings.h"
 
 
-/* Platform-specific time_t typedef */
-typedef time_t NPY_TIME_T;
-
-/*
- * Wraps `localtime` functionality for multiple platforms. This
- * converts a time value to a time structure in the local timezone.
- *
- * Returns 0 on success, -1 on failure.
- */
-static int get_localtime(NPY_TIME_T *ts, struct tm *tms) {
-    char *func_name = "<unknown>";
-#if defined(_WIN32)
-#if defined(_MSC_VER) && (_MSC_VER >= 1400)
-    if (localtime_s(tms, ts) != 0) {
-        func_name = "localtime_s";
-        goto fail;
-    }
-#elif defined(__GNUC__) && defined(NPY_MINGW_USE_CUSTOM_MSVCR)
-    if (_localtime64_s(tms, ts) != 0) {
-        func_name = "_localtime64_s";
-        goto fail;
-    }
-#else
-    struct tm *tms_tmp;
-    localtime_r(ts, tms_tmp);
-    if (tms_tmp == NULL) {
-        func_name = "localtime";
-        goto fail;
-    }
-    memcpy(tms, tms_tmp, sizeof(struct tm));
-#endif
-#else
-    if (localtime_r(ts, tms) == NULL) {
-        func_name = "localtime_r";
-        goto fail;
-    }
-#endif
-
-    return 0;
-
-fail:
-    PyErr_Format(PyExc_OSError,
-                 "Failed to use '%s' to convert "
-                 "to a local time",
-                 func_name);
-    return -1;
-}
-
-
 /*
  * Parses (almost) standard ISO 8601 date strings. The differences are:
  *
@@ -138,59 +89,6 @@ int parse_iso_8601_datetime(char *str, int len,
     out->month = 1;
     out->day = 1;
 
-    /*
-     * The string "today" means take today's date in local time, and
-     * convert it to a date representation. This date representation, if
-     * forced into a time unit, will be at midnight UTC.
-     * This is perhaps a little weird, but done so that the
-     * 'datetime64[D]' type produces the date you expect, rather than
-     * switching to an adjacent day depending on the current time and your
-     * timezone.
-     */
-    if (len == 5 && tolower(str[0]) == 't' && tolower(str[1]) == 'o' &&
-        tolower(str[2]) == 'd' && tolower(str[3]) == 'a' &&
-        tolower(str[4]) == 'y') {
-        NPY_TIME_T rawtime = 0;
-        struct tm tm_;
-
-        time(&rawtime);
-        if (get_localtime(&rawtime, &tm_) < 0) {
-            return -1;
-        }
-        out->year = tm_.tm_year + 1900;
-        out->month = tm_.tm_mon + 1;
-        out->day = tm_.tm_mday;
-
-        /*
-         * Indicate that this was a special value, and
-         * is a date (unit 'D').
-         */
-        if (out_local != NULL) {
-            *out_local = 0;
-        }
-
-        return 0;
-    }
-
-    /* The string "now" resolves to the current UTC time */
-    if (len == 3 && tolower(str[0]) == 'n' && tolower(str[1]) == 'o' &&
-        tolower(str[2]) == 'w') {
-        NPY_TIME_T rawtime = 0;
-
-        time(&rawtime);
-
-        /*
-         * Indicate that this was a special value, and
-         * use 's' because the time() function has resolution
-         * seconds.
-         */
-        if (out_local != NULL) {
-            *out_local = 0;
-        }
-
-        return convert_datetime_to_datetimestruct(PANDAS_FR_s, rawtime, out);
-    }
-
     substr = str;
     sublen = len;