CLN: Clean up of locale testing

ci/azure/posix.yml

@@ -23,14 +23,16 @@ jobs:
           ENV_FILE: ci/deps/azure-36-locale.yaml
           CONDA_PY: "36"
           PATTERN: "slow"
-          LOCALE_OVERRIDE: "zh_CN.UTF-8"
+          LANG: "zh_CN.GB2312"
+          LC_ALL: "zh_CN.GB2312"
           EXTRA_APT: "language-pack-zh-hans"
 
         py36_locale_slow:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
           CONDA_PY: "36"
           PATTERN: "not slow and not network"
-          LOCALE_OVERRIDE: "it_IT.UTF-8"
+          LANG: "it_IT.ISO-8859-1"
+          LC_ALL: "it_IT.ISO-8859-1"
 
         py36_32bit:
           ENV_FILE: ci/deps/azure-36-32bit.yaml
@@ -42,7 +44,8 @@ jobs:
           ENV_FILE: ci/deps/azure-37-locale.yaml
           CONDA_PY: "37"
           PATTERN: "not slow and not network"
-          LOCALE_OVERRIDE: "zh_CN.UTF-8"
+          LANG: "zh_CN.GB2312"
+          LC_ALL: "zh_CN.GB2312"
 
         py37_np_dev:
           ENV_FILE: ci/deps/azure-37-numpydev.yaml

ci/run_tests.sh

@@ -5,17 +5,6 @@
 # https://github.com/pytest-dev/pytest/issues/1075
 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
 
-if [ -n "$LOCALE_OVERRIDE" ]; then
-    export LC_ALL="$LOCALE_OVERRIDE"
-    export LANG="$LOCALE_OVERRIDE"
-    PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'`
-    if [[ "$LOCALE_OVERRIDE" != "$PANDAS_LOCALE" ]]; then
-        echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE"
-        # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed
-        # exit 1
-    fi
-fi
-
 if [[ "not network" == *"$PATTERN"* ]]; then
     export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
 fi

ci/setup_env.sh

@@ -1,15 +1,15 @@
 #!/bin/bash -e
 
 # edit the locale file if needed
-if [ -n "$LOCALE_OVERRIDE" ]; then
+if [ -n "$LC_ALL" ]; then
     echo "Adding locale to the first line of pandas/__init__.py"
     rm -f pandas/__init__.pyc
-    SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n"
+    SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n"
     sed -i "$SEDC" pandas/__init__.py
     echo "[head -4 pandas/__init__.py]"
     head -4 pandas/__init__.py
     echo
-    sudo locale-gen "$LOCALE_OVERRIDE"
+    sudo locale-gen "$LC_ALL"
 fi
 
 MINICONDA_DIR="$HOME/miniconda3"

pandas/tests/config/test_localization.py

@@ -8,6 +8,8 @@
 
 from pandas.compat import is_platform_windows
 
+import pandas as pd
+
 _all_locales = get_locales() or []
 _current_locale = locale.getlocale()
 
@@ -56,21 +58,21 @@ def test_get_locales_prefix():
 
 
 @_skip_if_only_one_locale
-def test_set_locale():
+@pytest.mark.parametrize(
+    "lang,enc",
+    [
+        ("it_CH", "UTF-8"),
+        ("en_US", "ascii"),
+        ("zh_CN", "GB2312"),
+        ("it_IT", "ISO-8859-1"),
+    ],
+)
+def test_set_locale(lang, enc):
     if all(x is None for x in _current_locale):
         # Not sure why, but on some Travis runs with pytest,
         #  getlocale() returned (None, None).
         pytest.skip("Current locale is not set.")
 
-    locale_override = os.environ.get("LOCALE_OVERRIDE", None)
-
-    if locale_override is None:
-        lang, enc = "it_CH", "UTF-8"
-    elif locale_override == "C":
-        lang, enc = "en_US", "ascii"
-    else:
-        lang, enc = locale_override.split(".")
-
     enc = codecs.lookup(enc).name
     new_locale = lang, enc
 
@@ -91,3 +93,13 @@ def test_set_locale():
     # Once we exit the "with" statement, locale should be back to what it was.
     current_locale = locale.getlocale()
     assert current_locale == _current_locale
+
+
+def test_encoding_detected():
+    system_locale = os.environ.get("LC_ALL")
+    if system_locale:
+        system_encoding = system_locale.split(".")[-1].upper().replace("-", "")
+    else:
+        system_encoding = "UTF8"
+
+    assert pd.options.display.encoding == system_encoding