merge with master again

Author: Dr-Irv

.circleci/config.yml

@@ -1,10 +1,6 @@
 version: 2
 jobs:
-
-  # --------------------------------------------------------------------------
-  # 1. py36_locale
-  # --------------------------------------------------------------------------
-  py36_locale:
+  build:
     docker:
       - image: continuumio/miniconda:latest
       # databases configuration
@@ -21,7 +17,7 @@ jobs:
 
     environment:
       JOB: "3.6_LOCALE"
-      ENV_FILE: "ci/circle-36-locale.yaml"
+      ENV_FILE: "ci/deps/circle-36-locale.yaml"
       LOCALE_OVERRIDE: "zh_CN.UTF-8"
       MINICONDA_DIR: /home/ubuntu/miniconda3
     steps:
@@ -30,13 +26,13 @@ jobs:
           name: build
           command: |
             ./ci/circle/install_circle.sh
-            ./ci/circle/show_circle.sh
+            export PATH="$MINICONDA_DIR/bin:$PATH"
+            source activate pandas-dev
+            python -c "import pandas; pandas.show_versions();"
       - run:
           name: test
-          command: ./ci/circle/run_circle.sh  --skip-slow --skip-network
-
-workflows:
-  version: 2
-  build_and_test:
-    jobs:
-      - py36_locale
+          command: |
+            export PATH="$MINICONDA_DIR/bin:$PATH"
+            source activate pandas-dev
+            echo "pytest --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml --skip-slow --skip-network pandas"
+            pytest       --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml --skip-slow --skip-network pandas

.gitignore

@@ -109,6 +109,5 @@ doc/build/html/index.html
 # Windows specific leftover:
 doc/tmp.sv
 doc/source/styled.xlsx
-doc/source/templates/
 env/
 doc/source/savefig/

.pep8speaks.yml

@@ -13,7 +13,6 @@ pycodestyle:
         - W503,  # line break before binary operator
         - W504,  # line break after binary operator
         - E402,  # module level import not at top of file
-        - E722,  # do not use bare except
         - E731,  # do not assign a lambda expression, use a def
         - C406,  # Unnecessary list literal - rewrite as a dict literal.
         - C408,  # Unnecessary dict call - rewrite as a literal.

.travis.yml

@@ -23,7 +23,7 @@ env:
 
 git:
     # for cloning
-    depth: 1000
+    depth: 1500
 
 matrix:
     fast_finish: true
@@ -34,28 +34,28 @@ matrix:
     include:
     - dist: trusty
       env:
-        - JOB="3.7" ENV_FILE="ci/travis-37.yaml" TEST_ARGS="--skip-slow --skip-network"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" TEST_ARGS="--skip-slow --skip-network"
 
     - dist: trusty
       env:
-        - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true
+        - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/deps/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true
       addons:
         apt:
           packages:
           - language-pack-zh-hans
     - dist: trusty
       env:
-        - JOB="2.7" ENV_FILE="ci/travis-27.yaml" TEST_ARGS="--skip-slow"
+        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" TEST_ARGS="--skip-slow"
       addons:
         apt:
           packages:
           - python-gtk2
     - dist: trusty
       env:
-        - JOB="3.6, lint, coverage" ENV_FILE="ci/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true
+        - JOB="3.6, lint, coverage" ENV_FILE="ci/deps/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true
     - dist: trusty
       env:
-        - JOB="3.7, NumPy dev" ENV_FILE="ci/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
+        - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
       addons:
         apt:
           packages:
@@ -64,19 +64,19 @@ matrix:
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6, slow" ENV_FILE="ci/travis-36-slow.yaml" SLOW=true
+        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true
 
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6, doc" ENV_FILE="ci/travis-36-doc.yaml" DOC=true
+        - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true
     allow_failures:
       - dist: trusty
         env:
-          - JOB="3.6, slow" ENV_FILE="ci/travis-36-slow.yaml" SLOW=true
+          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true
       - dist: trusty
         env:
-          - JOB="3.6, doc" ENV_FILE="ci/travis-36-doc.yaml" DOC=true
+          - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true
 
 before_install:
   - echo "before_install"
@@ -105,6 +105,7 @@ before_script:
 
 script:
   - echo "script start"
+  - source activate pandas-dev
   - ci/run_build_docs.sh
   - ci/script_single.sh
   - ci/script_multi.sh
@@ -115,7 +116,7 @@ after_success:
 
 after_script:
   - echo "after_script start"
-  - source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+  - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
   - if [ -e test-data-single.xml ]; then
     ci/print_skipped.py test-data-single.xml;
     fi

Makefile

@@ -13,7 +13,7 @@ build: clean_pyc
 	python setup.py build_ext --inplace
 
 lint-diff:
-	git diff master --name-only -- "*.py" | grep -E "pandas|scripts" | xargs flake8
+	git diff upstream/master --name-only -- "*.py" | xargs flake8
 
 develop: build
 	-python setup.py develop

asv_bench/benchmarks/reshape.py

@@ -49,21 +49,33 @@ def time_unstack(self):
 
 class Unstack(object):
 
-    def setup(self):
+    params = ['int', 'category']
+
+    def setup(self, dtype):
         m = 100
         n = 1000
 
         levels = np.arange(m)
         index = MultiIndex.from_product([levels] * 2)
         columns = np.arange(n)
-        values = np.arange(m * m * n).reshape(m * m, n)
+        if dtype == 'int':
+            values = np.arange(m * m * n).reshape(m * m, n)
+        else:
+            # the category branch is ~20x slower than int. So we
+            # cut down the size a bit. Now it's only ~3x slower.
+            n = 50
+            columns = columns[:n]
+            indices = np.random.randint(0, 52, size=(m * m, n))
+            values = np.take(list(string.ascii_letters), indices)
+            values = [pd.Categorical(v) for v in values.T]
+
         self.df = DataFrame(values, index, columns)
         self.df2 = self.df.iloc[:-1]
 
-    def time_full_product(self):
+    def time_full_product(self, dtype):
         self.df.unstack()
 
-    def time_without_last_row(self):
+    def time_without_last_row(self, dtype):
         self.df2.unstack()
 
 

asv_bench/benchmarks/timeseries.py

@@ -1,5 +1,6 @@
 from datetime import timedelta
 
+import dateutil
 import numpy as np
 from pandas import to_datetime, date_range, Series, DataFrame, period_range
 from pandas.tseries.frequencies import infer_freq
@@ -57,7 +58,10 @@ def time_to_pydatetime(self, index_type):
 
 class TzLocalize(object):
 
-    def setup(self):
+    params = [None, 'US/Eastern', 'UTC', dateutil.tz.tzutc()]
+    param_names = 'tz'
+
+    def setup(self, tz):
         dst_rng = date_range(start='10/29/2000 1:00:00',
                              end='10/29/2000 1:59:59', freq='S')
         self.index = date_range(start='10/29/2000',
@@ -68,8 +72,8 @@ def setup(self):
                                                   end='10/29/2000 3:00:00',
                                                   freq='S'))
 
-    def time_infer_dst(self):
-        self.index.tz_localize('US/Eastern', ambiguous='infer')
+    def time_infer_dst(self, tz):
+        self.index.tz_localize(tz, ambiguous='infer')
 
 
 class ResetIndex(object):
@@ -377,15 +381,35 @@ def time_dup_string_tzoffset_dates(self, cache):
 
 class DatetimeAccessor(object):
 
-    def setup(self):
+    params = [None, 'US/Eastern', 'UTC', dateutil.tz.tzutc()]
+    param_names = 'tz'
+
+    def setup(self, tz):
         N = 100000
-        self.series = Series(date_range(start='1/1/2000', periods=N, freq='T'))
+        self.series = Series(
+            date_range(start='1/1/2000', periods=N, freq='T', tz=tz)
+        )
 
-    def time_dt_accessor(self):
+    def time_dt_accessor(self, tz):
         self.series.dt
 
-    def time_dt_accessor_normalize(self):
+    def time_dt_accessor_normalize(self, tz):
         self.series.dt.normalize()
 
+    def time_dt_accessor_month_name(self, tz):
+        self.series.dt.month_name()
+
+    def time_dt_accessor_day_name(self, tz):
+        self.series.dt.day_name()
+
+    def time_dt_accessor_time(self, tz):
+        self.series.dt.time
+
+    def time_dt_accessor_date(self, tz):
+        self.series.dt.date
+
+    def time_dt_accessor_year(self, tz):
+        self.series.dt.year
+
 
 from .pandas_vb_common import setup  # noqa: F401

asv_bench/benchmarks/timestamp.py

@@ -2,6 +2,7 @@
 
 from pandas import Timestamp
 import pytz
+import dateutil
 
 
 class TimestampConstruction(object):
@@ -29,7 +30,8 @@ def time_fromtimestamp(self):
 
 
 class TimestampProperties(object):
-    _tzs = [None, pytz.timezone('Europe/Amsterdam')]
+    _tzs = [None, pytz.timezone('Europe/Amsterdam'), pytz.UTC,
+            dateutil.tz.tzutc()]
     _freqs = [None, 'B']
     params = [_tzs, _freqs]
     param_names = ['tz', 'freq']
@@ -87,7 +89,8 @@ def time_microsecond(self, tz, freq):
 
 
 class TimestampOps(object):
-    params = [None, 'US/Eastern']
+    params = [None, 'US/Eastern', pytz.UTC,
+              dateutil.tz.tzutc()]
     param_names = ['tz']
 
     def setup(self, tz):
@@ -102,6 +105,17 @@ def time_replace_None(self, tz):
     def time_to_pydatetime(self, tz):
         self.ts.to_pydatetime()
 
+    def time_normalize(self, tz):
+        self.ts.normalize()
+
+    def time_tz_convert(self, tz):
+        if self.ts.tz is not None:
+            self.ts.tz_convert(tz)
+
+    def time_tz_localize(self, tz):
+        if self.ts.tz is None:
+            self.ts.tz_localize(tz)
+
 
 class TimestampAcrossDst(object):
     def setup(self):

ci/azure/linux.yml

@@ -10,23 +10,21 @@ jobs:
     maxParallel: 11
     matrix:
       py27_np_19:
-        ENV_FILE: ci/azure-27-compat.yaml
+        ENV_FILE: ci/deps/azure-27-compat.yaml
         CONDA_PY: "27"
-        CONDA_ENV: pandas
         TEST_ARGS: "--skip-slow --skip-network"
 
-      py36_locale:
-        ENV_FILE: ci/azure-37-locale.yaml
+      py37_locale:
+        ENV_FILE: ci/deps/azure-37-locale.yaml
         CONDA_PY: "37"
-        CONDA_ENV: pandas
         TEST_ARGS: "--skip-slow --skip-network"
         LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
       py36_locale_slow:
-        ENV_FILE: ci/azure-36-locale_slow.yaml
+        ENV_FILE: ci/deps/azure-36-locale_slow.yaml
         CONDA_PY: "36"
-        CONDA_ENV: pandas
         TEST_ARGS: "--only-slow --skip-network"
+        LOCALE_OVERRIDE: "it_IT.UTF-8"
 
   steps:
     - script: |
@@ -39,18 +37,45 @@ jobs:
       displayName: 'Before Install'
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
+        source activate pandas-dev
         ci/incremental/build.sh
       displayName: 'Build'
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
+        source activate pandas-dev
         ci/script_single.sh
         ci/script_multi.sh
         echo "[Test done]"
       displayName: 'Test'
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
-        source activate pandas && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
+        source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
     - task: PublishTestResults@2
       inputs:
         testResultsFiles: 'test-data-*.xml'
-        testRunTitle: 'Linux'
+        testRunTitle: 'Linux'
+    - powershell: |
+        $junitXml = "test-data-single.xml"
+        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data-single"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+
+        $junitXmlMulti = "test-data-multiple.xml"
+        $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0)
+        {
+          Write-Host "No test failures in test-data-multi"
+        }
+        else
+        {
+          # note that this will produce $LASTEXITCODE=1
+          Write-Error "$($matches[1]) tests failed"
+        }
+      displayName: Check for test failures