Merge branch 'master' into GH13410-ENHunion_categoricals

Author: js3711

.travis.yml

@@ -132,6 +132,7 @@ matrix:
       env:
         - PYTHON_VERSION=3.4
         - JOB_NAME: "34_nslow"
+        - LOCALE_OVERRIDE="zh_CN.UTF-8"
         - NOSE_ARGS="not slow and not disabled"
         - FULL_DEPS=true
         - CLIPBOARD=xsel
@@ -141,6 +142,7 @@ matrix:
         apt:
           packages:
           - xsel
+          - language-pack-zh-hans
 #    In allow_failures
     - python: 3.4
       env:
@@ -229,6 +231,7 @@ matrix:
         env:
           - PYTHON_VERSION=3.4
           - JOB_NAME: "34_nslow"
+          - LOCALE_OVERRIDE="zh_CN.UTF-8"
           - NOSE_ARGS="not slow and not disabled"
           - FULL_DEPS=true
           - CLIPBOARD=xsel
@@ -238,6 +241,7 @@ matrix:
           apt:
             packages:
             - xsel
+            - language-pack-zh-hans
       - python: 3.5
         env:
           - PYTHON_VERSION=3.5

asv_bench/benchmarks/frame_methods.py

@@ -433,6 +433,20 @@ def time_frame_from_records_generator_nrows(self):
 
 
 
+#-----------------------------------------------------------------------------
+# nunique
+
+class frame_nunique(object):
+
+    def setup(self):
+        self.data = np.random.randn(10000, 1000)
+        self.df = DataFrame(self.data)
+
+    def time_frame_nunique(self):
+        self.df.nunique()
+
+
+
 #-----------------------------------------------------------------------------
 # duplicated
 

asv_bench/benchmarks/groupby.py

@@ -251,6 +251,22 @@ def time_groupby_int_count(self):
         self.df.groupby(['key1', 'key2']).count()
 
 
+#----------------------------------------------------------------------
+# nunique() speed
+
+class groupby_nunique(object):
+
+    def setup(self):
+        self.n = 10000
+        self.df = DataFrame({'key1': randint(0, 500, size=self.n),
+                             'key2': randint(0, 100, size=self.n),
+                             'ints': randint(0, 1000, size=self.n),
+                             'ints2': randint(0, 1000, size=self.n), })
+
+    def time_groupby_nunique(self):
+        self.df.groupby(['key1', 'key2']).nunique()
+
+
 #----------------------------------------------------------------------
 # group with different functions per column
 

ci/build_docs.sh

@@ -22,8 +22,8 @@ if [ x"$DOC_BUILD" != x"" ]; then
     echo "Will build docs"
 
     source activate pandas
-    conda install -n pandas -c r r rpy2 --yes
 
+    # install sudo deps
     time sudo apt-get $APT_ARGS install dvipng texlive-latex-base texlive-latex-extra
 
     mv "$TRAVIS_BUILD_DIR"/doc /tmp

ci/install_travis.sh

@@ -16,47 +16,48 @@
 function edit_init()
 {
     if [ -n "$LOCALE_OVERRIDE" ]; then
-        echo "Adding locale to the first line of pandas/__init__.py"
+        echo "[Adding locale to the first line of pandas/__init__.py]"
         rm -f pandas/__init__.pyc
         sedc="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n"
         sed -i "$sedc" pandas/__init__.py
-        echo "head -4 pandas/__init__.py"
+        echo "[head -4 pandas/__init__.py]"
         head -4 pandas/__init__.py
         echo
     fi
 }
 
+echo "[install_travis]"
 edit_init
 
 home_dir=$(pwd)
-echo "home_dir: [$home_dir]"
+echo "[home_dir: $home_dir]"
 
 MINICONDA_DIR="$HOME/miniconda3"
 
 if [ -d "$MINICONDA_DIR" ] && [ -e "$MINICONDA_DIR/bin/conda" ] && [ "$USE_CACHE" ]; then
-    echo "Miniconda install already present from cache: $MINICONDA_DIR"
+    echo "[Miniconda install already present from cache: $MINICONDA_DIR]"
 
     conda config --set always_yes yes --set changeps1 no || exit 1
-    echo "update conda"
+    echo "[update conda]"
     conda update -q conda || exit 1
 
     # Useful for debugging any issues with conda
     conda info -a || exit 1
 
     # set the compiler cache to work
     if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
-        echo "Using ccache"
+        echo "[Using ccache]"
         export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH
         gcc=$(which gcc)
-        echo "gcc: $gcc"
+        echo "[gcc: $gcc]"
         ccache=$(which ccache)
-        echo "ccache: $ccache"
+        echo "[ccache: $ccache]"
         export CC='ccache gcc'
     fi
 
 else
-    echo "Using clean Miniconda install"
-    echo "Not using ccache"
+    echo "[Using clean Miniconda install]"
+    echo "[Not using ccache]"
     rm -rf "$MINICONDA_DIR"
     # install miniconda
     if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
@@ -66,14 +67,14 @@ else
     fi
     bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1
 
-    echo "update conda"
+    echo "[update conda]"
     conda config --set ssl_verify false || exit 1
     conda config --set always_yes true --set changeps1 false || exit 1
     conda update -q conda
 
     # add the pandas channel to take priority
     # to add extra packages
-    echo "add channels"
+    echo "[add channels]"
     conda config --add channels pandas || exit 1
     conda config --remove channels defaults || exit 1
     conda config --add channels defaults || exit 1
@@ -103,13 +104,19 @@ else
 fi
 
 # build deps
+echo "[build installs]"
 REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build"
-
-# install deps
 if [ -e ${REQ} ]; then
     time conda install -n pandas --file=${REQ} || exit 1
 fi
 
+# may have addtl installation instructions for this build
+echo "[build addtl installs]"
+REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build.sh"
+if [ -e ${REQ} ]; then
+    time bash $REQ || exit 1
+fi
+
 source activate pandas
 
 if [ "$BUILD_TEST" ]; then
@@ -122,39 +129,40 @@ if [ "$BUILD_TEST" ]; then
 else
 
     # build but don't install
-    echo "build em"
+    echo "[build em]"
     time python setup.py build_ext --inplace || exit 1
 
     # we may have run installations
-    echo "conda installs"
+    echo "[conda installs]"
     REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.run"
     if [ -e ${REQ} ]; then
         time conda install -n pandas --file=${REQ} || exit 1
     fi
 
     # we may have additional pip installs
-    echo "pip installs"
+    echo "[pip installs]"
     REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.pip"
     if [ -e ${REQ} ]; then
-       pip install --upgrade -r $REQ
+       pip install -r $REQ
     fi
 
     # may have addtl installation instructions for this build
+    echo "[addtl installs]"
     REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.sh"
     if [ -e ${REQ} ]; then
         time bash $REQ || exit 1
     fi
 
     # remove any installed pandas package
     # w/o removing anything else
-    echo "removing installed pandas"
+    echo "[removing installed pandas]"
     conda remove pandas --force
 
     # install our pandas
-    echo "running setup.py develop"
+    echo "[running setup.py develop]"
     python setup.py develop  || exit 1
 
 fi
 
-echo "done"
+echo "[done]"
 exit 0

ci/ironcache/get.py

@@ -8,10 +8,9 @@ RET=0
 
 if [ "$LINT" ]; then
 
-    # pandas/rpy is deprecated and will be removed.
     # pandas/src is C code, so no need to search there.
     echo "Linting  *.py"
-    flake8 pandas --filename=*.py --exclude pandas/rpy,pandas/src
+    flake8 pandas --filename=*.py --exclude pandas/src
     if [ $? -ne "0" ]; then
         RET=1
     fi

ci/ironcache/put.py

@@ -20,4 +20,4 @@ html5lib=1.0b2
 beautiful-soup=4.2.1
 statsmodels
 jinja2=2.8
-xarray
+xarray=0.8.0

ci/lint.sh

@@ -1,4 +1,4 @@
 python-dateutil
 pytz
-numpy=1.9.3
+numpy=1.10*
 cython

ci/requirements-2.7.run

@@ -1,6 +1,6 @@
 python-dateutil
 pytz
-numpy=1.9.3
+numpy=1.10*
 openpyxl
 xlsxwriter
 xlrd

ci/requirements-3.4_SLOW.build

@@ -4,4 +4,4 @@ source activate pandas
 
 echo "install 34_slow"
 
-conda install -n pandas -c conda-forge/label/rc -c conda-forge matplotlib
+conda install -n pandas -c conda-forge matplotlib

ci/requirements-3.4_SLOW.run

@@ -0,0 +1 @@
+xarray==0.9.1

ci/requirements-3.4_SLOW.sh

@@ -16,6 +16,5 @@ bottleneck
 sqlalchemy
 pymysql
 psycopg2
-xarray
 s3fs
 beautifulsoup4

ci/requirements-3.5.pip

@@ -2,6 +2,8 @@
 
 source activate pandas
 
-echo "install DOC_BUILD"
+echo "[install DOC_BUILD deps]"
 
 conda install -n pandas -c conda-forge feather-format
+
+conda install -n pandas -c r r rpy2 --yes

ci/requirements-3.5.run

@@ -7,11 +7,6 @@ echo "install numpy master wheel"
 # remove the system installed numpy
 pip uninstall numpy -y
 
-# we need these for numpy
-
-# these wheels don't play nice with the conda libgfortran / openblas
-# time conda install -n pandas libgfortran openblas || exit 1
-
 # install numpy wheel from master
 pip install --pre --upgrade --no-index --timeout=60 --trusted-host travis-dev-wheels.scipy.org -f http://travis-dev-wheels.scipy.org/ numpy scipy
 

ci/requirements-3.5_DOC_BUILD.sh

@@ -1 +0,0 @@
-xlwt