TST: Xfails in Python 3.10 CI

[lithomas1]

A list of failures for pandas on Python 3.10.
Note: Some of these are flaky.

• TestUnique.test_first_nan_kept

________________________ TestUnique.test_first_nan_kept ________________________

self = <pandas.tests.test_algos.TestUnique object at 0x7fe611cc19c0>

def test_first_nan_kept(self):
    # GH 22295
    # create different nans from bit-patterns:
    bits_for_nan1 = 0xFFF8000000000001
    bits_for_nan2 = 0x7FF8000000000001
    NAN1 = struct.unpack("d", struct.pack("=Q", bits_for_nan1))[0]
    NAN2 = struct.unpack("d", struct.pack("=Q", bits_for_nan2))[0]
    assert NAN1 != NAN1
    assert NAN2 != NAN2
    for el_type in [np.float64, object]:
        a = np.array([NAN1, NAN2], dtype=el_type)
        result = pd.unique(a)

      assert result.size == 1

E assert 2 == 1
E + where 2 = array([nan, nan], dtype=object).size

pandas/tests/test_algos.py:798: AssertionError

• TestIsin.test_different_nans

_________________________ TestIsin.test_different_nans _________________________

self = <pandas.tests.test_algos.TestIsin object at 0x7fe611cc2b30>

def test_different_nans(self):
    # GH 22160
    # all nans are handled as equivalent

    comps = [float("nan")]
    values = [float("nan")]
    assert comps[0] is not values[0]  # different nan-objects

    # as list of python-objects:
    result = algos.isin(comps, values)
    tm.assert_numpy_array_equal(np.array([True]), result)

    # as object-array:
    result = algos.isin(
        np.asarray(comps, dtype=object), np.asarray(values, dtype=object)
    )

  tm.assert_numpy_array_equal(np.array([True]), result)

pandas/tests/test_algos.py:1007:

---

left = array([ True]), right = array([False]), err_msg = None

def _raise(left, right, err_msg):
    if err_msg is None:
        if left.shape != right.shape:
            raise_assert_detail(
                obj, f"{obj} shapes are different", left.shape, right.shape
            )

        diff = 0
        for left_arr, right_arr in zip(left, right):
            # count up differences
            if not array_equivalent(left_arr, right_arr, strict_nan=strict_nan):
                diff += 1

        diff = diff * 100.0 / left.size
        msg = f"{obj} values are different ({np.round(diff, 5)} %)"

      raise_assert_detail(obj, msg, left, right, index_values=index_values)

E AssertionError: numpy array are different
E
E numpy array values are different (100.0 %)
E [left]: [True]
E [right]: [False]

pandas/_testing/asserters.py:726: AssertionError

• TestIsin.test_different_nan_objects

_____________________ TestIsin.test_different_nan_objects ______________________

self = <pandas.tests.test_algos.TestIsin object at 0x7fe61167ff40>

def test_different_nan_objects(self):
    # GH 22119
    comps = np.array(["nan", np.nan * 1j, float("nan")], dtype=object)
    vals = np.array([float("nan")], dtype=object)
    expected = np.array([False, False, True])
    result = algos.isin(comps, vals)

  tm.assert_numpy_array_equal(expected, result)

pandas/tests/test_algos.py:1039:

---

left = array([False, False, True]), right = array([False, False, False])
err_msg = None

def _raise(left, right, err_msg):
    if err_msg is None:
        if left.shape != right.shape:
            raise_assert_detail(
                obj, f"{obj} shapes are different", left.shape, right.shape
            )

        diff = 0
        for left_arr, right_arr in zip(left, right):
            # count up differences
            if not array_equivalent(left_arr, right_arr, strict_nan=strict_nan):
                diff += 1

        diff = diff * 100.0 / left.size
        msg = f"{obj} values are different ({np.round(diff, 5)} %)"

      raise_assert_detail(obj, msg, left, right, index_values=index_values)

E AssertionError: numpy array are different
E
E numpy array values are different (33.33333 %)
E [left]: [False, False, True]
E [right]: [False, False, False]

pandas/_testing/asserters.py:726: AssertionError

• test_hash_equal

_________________________ test_hash_equal[a2-b2-True] __________________________

a = Sparse[float64, nan], b = Sparse[float64, nan], expected = True

@pytest.mark.parametrize(
    "a, b, expected",
    [
        (SparseDtype(float, 0.0), SparseDtype(np.dtype("float"), 0.0), True),
        (SparseDtype(int, 0), SparseDtype(int, 0), True),
        (SparseDtype(float, float("nan")), SparseDtype(float, np.nan), True),
        (SparseDtype(float, 0), SparseDtype(float, np.nan), False),
        (SparseDtype(int, 0.0), SparseDtype(float, 0.0), False),
    ],
)
def test_hash_equal(a, b, expected):
    result = a == b
    assert result is expected

    result = hash(a) == hash(b)

  assert result is expected

E assert False is True

pandas/tests/arrays/sparse/test_dtype.py:141: AssertionError

• TestReshaping.test_stack

____________________ TestReshaping.test_stack[nan-columns0] ____________________

self = <pandas.tests.extension.test_sparse.TestReshaping object at 0x7fe5fc4a6d10>
data = [0.6069529775354311, 0.6532416455715819, nan, 0.45409596635270144, 0.10948849972327801, nan, 0.08987488776518548, 0.19...66, 67, 69, 70, 72, 73, 75,
76, 78, 79, 81, 82, 84, 85, 87, 88, 90, 91, 93, 94, 96, 97, 99],
dtype=int32)

columns = ['A', 'B']

@pytest.mark.parametrize(
    "columns",
    [
        ["A", "B"],
        pd.MultiIndex.from_tuples(
            [("A", "a"), ("A", "b")], names=["outer", "inner"]
        ),
    ],
)
def test_stack(self, data, columns):
    df = pd.DataFrame({"A": data[:5], "B": data[:5]})
    df.columns = columns
    result = df.stack()
    expected = df.astype(object).stack()
    # we need a second astype(object), in case the constructor inferred
    # object -> specialized, as is done for period.
    expected = expected.astype(object)

    if isinstance(expected, pd.Series):
        assert result.dtype == df.iloc[:, 0].dtype
    else:
        assert all(result.dtypes == df.iloc[:, 0].dtype)

    result = result.astype(object)

  self.assert_equal(result, expected)

pandas/tests/extension/base/reshaping.py:270:

---

cls = <class 'pandas.tests.extension.test_sparse.TestReshaping'>
left = 0 A 0.606953
B 0.606953
1 A 0.653242
B 0.653242
3 A 0.454096
B 0.454096
4 A 0.109488
B 0.109488
dtype: Sparse[object, nan]
right = 0 A 0.606953
B 0.606953
1 A 0.653242
B 0.653242
3 A 0.454096
B 0.454096
4 A 0.109488
B 0.109488
dtype: object
kwargs = {}

@classmethod
def assert_equal(cls, left, right, **kwargs):

  return tm.assert_equal(left, right, **kwargs)

E AssertionError: Attributes of Series are different
E
E Attribute "dtype" are different
E [left]: Sparse[object, nan]
E [right]: object

pandas/tests/extension/base/base.py:9: AssertionError

____________________ TestReshaping.test_stack[nan-columns1] ____________________

self = <pandas.tests.extension.test_sparse.TestReshaping object at 0x7fe5fbea8a30>
data = [0.08642533151098042, 0.31948867044255314, nan, 0.7649119528014422, 0.8562776453950798, nan, 0.0026917407695283835, 0....66, 67, 69, 70, 72, 73, 75,
76, 78, 79, 81, 82, 84, 85, 87, 88, 90, 91, 93, 94, 96, 97, 99],
dtype=int32)

columns = MultiIndex([('A', 'a'),
('A', 'b')],
names=['outer', 'inner'])

@pytest.mark.parametrize(
    "columns",
    [
        ["A", "B"],
        pd.MultiIndex.from_tuples(
            [("A", "a"), ("A", "b")], names=["outer", "inner"]
        ),
    ],
)
def test_stack(self, data, columns):
    df = pd.DataFrame({"A": data[:5], "B": data[:5]})
    df.columns = columns
    result = df.stack()
    expected = df.astype(object).stack()
    # we need a second astype(object), in case the constructor inferred
    # object -> specialized, as is done for period.
    expected = expected.astype(object)

    if isinstance(expected, pd.Series):
        assert result.dtype == df.iloc[:, 0].dtype
    else:
        assert all(result.dtypes == df.iloc[:, 0].dtype)

    result = result.astype(object)

  self.assert_equal(result, expected)

pandas/tests/extension/base/reshaping.py:270:

---

cls = <class 'pandas.tests.extension.test_sparse.TestReshaping'>
left = outer A
inner
0 a 0.086425
b 0.086425
1 a 0.319489
b 0.319489
3 a 0.764912
b 0.764912
4 a 0.856278
b 0.856278
right = outer A
inner
0 a 0.086425
b 0.086425
1 a 0.319489
b 0.319489
3 a 0.764912
b 0.764912
4 a 0.856278
b 0.856278
kwargs = {}

@classmethod
def assert_equal(cls, left, right, **kwargs):

  return tm.assert_equal(left, right, **kwargs)

E AssertionError: Attributes of DataFrame.iloc[:, 0] (column name="A") are different
E
E Attribute "dtype" are different
E [left]: Sparse[object, nan]
E [right]: object

pandas/tests/extension/base/base.py:9: AssertionError

• TestDataFrameSelectReindex.test_reindex_nan

_________________ TestDataFrameSelectReindex.test_reindex_nan __________________

self = <pandas.tests.frame.methods.test_reindex.TestDataFrameSelectReindex object at 0x7fe5f9d03f40>

def test_reindex_nan(self):
    df = DataFrame(
        [[1, 2], [3, 5], [7, 11], [9, 23]],
        index=[2, np.nan, 1, 5],
        columns=["joe", "jim"],
    )

    i, j = [np.nan, 5, 5, np.nan, 1, 2, np.nan], [1, 3, 3, 1, 2, 0, 1]
    tm.assert_frame_equal(df.reindex(i), df.iloc[j])

    df.index = df.index.astype("object")

  tm.assert_frame_equal(df.reindex(i), df.iloc[j], check_index_type=False)

E AssertionError: Attributes of DataFrame.iloc[:, 0] (column name="joe") are different
E
E Attribute "dtype" are different
E [left]: float64
E [right]: int64

pandas/tests/frame/methods/test_reindex.py:472: AssertionError

• TestIndex.test_isin_nan_common_object

_____________ TestIndex.test_isin_nan_common_object[float0-float1] _____________

self = <pandas.tests.indexes.test_base.TestIndex object at 0x7fe5f6a4d930>
nulls_fixture = nan, nulls_fixture2 = nan

def test_isin_nan_common_object(self, nulls_fixture, nulls_fixture2):
    # Test cartesian product of null fixtures and ensure that we don't
    # mangle the various types (save a corner case with PyPy)

    # all nans are the same
    if (
        isinstance(nulls_fixture, float)
        and isinstance(nulls_fixture2, float)
        and math.isnan(nulls_fixture)
        and math.isnan(nulls_fixture2)
    ):

      tm.assert_numpy_array_equal(

            Index(["a", nulls_fixture]).isin([nulls_fixture2]),
            np.array([False, True]),
        )

pandas/tests/indexes/test_base.py:1006:

---

left = array([False, False]), right = array([False, True]), err_msg = None

def _raise(left, right, err_msg):
    if err_msg is None:
        if left.shape != right.shape:
            raise_assert_detail(
                obj, f"{obj} shapes are different", left.shape, right.shape
            )

        diff = 0
        for left_arr, right_arr in zip(left, right):
            # count up differences
            if not array_equivalent(left_arr, right_arr, strict_nan=strict_nan):
                diff += 1

        diff = diff * 100.0 / left.size
        msg = f"{obj} values are different ({np.round(diff, 5)} %)"

      raise_assert_detail(obj, msg, left, right, index_values=index_values)

E AssertionError: numpy array are different
E
E numpy array values are different (50.0 %)
E [left]: [False, False]
E [right]: [False, True]

pandas/_testing/asserters.py:726: AssertionError
_____________ TestIndex.test_isin_nan_common_object[float1-float0] _____________

self = <pandas.tests.indexes.test_base.TestIndex object at 0x7fe5f5e186a0>
nulls_fixture = nan, nulls_fixture2 = nan

def test_isin_nan_common_object(self, nulls_fixture, nulls_fixture2):
    # Test cartesian product of null fixtures and ensure that we don't
    # mangle the various types (save a corner case with PyPy)

    # all nans are the same
    if (
        isinstance(nulls_fixture, float)
        and isinstance(nulls_fixture2, float)
        and math.isnan(nulls_fixture)
        and math.isnan(nulls_fixture2)
    ):

      tm.assert_numpy_array_equal(

            Index(["a", nulls_fixture]).isin([nulls_fixture2]),
            np.array([False, True]),
        )

pandas/tests/indexes/test_base.py:1006:

---

left = array([False, False]), right = array([False, True]), err_msg = None

def _raise(left, right, err_msg):
    if err_msg is None:
        if left.shape != right.shape:
            raise_assert_detail(
                obj, f"{obj} shapes are different", left.shape, right.shape
            )

        diff = 0
        for left_arr, right_arr in zip(left, right):
            # count up differences
            if not array_equivalent(left_arr, right_arr, strict_nan=strict_nan):
                diff += 1

        diff = diff * 100.0 / left.size
        msg = f"{obj} values are different ({np.round(diff, 5)} %)"

      raise_assert_detail(obj, msg, left, right, index_values=index_values)

E AssertionError: numpy array are different
E
E numpy array values are different (50.0 %)
E [left]: [False, False]
E [right]: [False, True]

pandas/_testing/asserters.py:726: AssertionError
_____________ TestIndex.test_isin_nan_common_object[float1-float1] _____________

self = <pandas.tests.indexes.test_base.TestIndex object at 0x7fe5f6ab3eb0>
nulls_fixture = nan, nulls_fixture2 = nan

def test_isin_nan_common_object(self, nulls_fixture, nulls_fixture2):
    # Test cartesian product of null fixtures and ensure that we don't
    # mangle the various types (save a corner case with PyPy)

    # all nans are the same
    if (
        isinstance(nulls_fixture, float)
        and isinstance(nulls_fixture2, float)
        and math.isnan(nulls_fixture)
        and math.isnan(nulls_fixture2)
    ):

      tm.assert_numpy_array_equal(

            Index(["a", nulls_fixture]).isin([nulls_fixture2]),
            np.array([False, True]),
        )

pandas/tests/indexes/test_base.py:1006:

---

left = array([False, False]), right = array([False, True]), err_msg = None

def _raise(left, right, err_msg):
    if err_msg is None:
        if left.shape != right.shape:
            raise_assert_detail(
                obj, f"{obj} shapes are different", left.shape, right.shape
            )

        diff = 0
        for left_arr, right_arr in zip(left, right):
            # count up differences
            if not array_equivalent(left_arr, right_arr, strict_nan=strict_nan):
                diff += 1

        diff = diff * 100.0 / left.size
        msg = f"{obj} values are different ({np.round(diff, 5)} %)"

      raise_assert_detail(obj, msg, left, right, index_values=index_values)

E AssertionError: numpy array are different
E
E numpy array values are different (50.0 %)
E [left]: [False, False]
E [right]: [False, True]

pandas/_testing/asserters.py:726: AssertionError

• TestUltraJSONTests.test_invalid_double_precision

_____________ TestUltraJSONTests.test_invalid_double_precision[9] ______________

self = <pandas.tests.io.json.test_ujson.TestUltraJSONTests object at 0x7fe5ee07d780>
invalid_val = '9'

@pytest.mark.parametrize("invalid_val", [20, -1, "9", None])
def test_invalid_double_precision(self, invalid_val):
    double_input = 30.12345678901234567890
    expected_exception = ValueError if isinstance(invalid_val, int) else TypeError
    msg = (
        r"Invalid value '.*' for option 'double_precision', max is '15'|"
        r"an integer is required \(got type "
    )
    with pytest.raises(expected_exception, match=msg):

      ujson.encode(double_input, double_precision=invalid_val)

E TypeError: 'str' object cannot be interpreted as an integer

pandas/tests/io/json/test_ujson.py:260: TypeError

During handling of the above exception, another exception occurred:

self = <pandas.tests.io.json.test_ujson.TestUltraJSONTests object at 0x7fe5ee07d780>
invalid_val = '9'

@pytest.mark.parametrize("invalid_val", [20, -1, "9", None])
def test_invalid_double_precision(self, invalid_val):
    double_input = 30.12345678901234567890
    expected_exception = ValueError if isinstance(invalid_val, int) else TypeError
    msg = (
        r"Invalid value '.*' for option 'double_precision', max is '15'|"
        r"an integer is required \(got type "
    )

  with pytest.raises(expected_exception, match=msg):

E AssertionError: Regex pattern "Invalid value '.*' for option 'double_precision', max is '15'|an integer is required \(got type " does not match "'str' object cannot be interpreted as an integer".

pandas/tests/io/json/test_ujson.py:259: AssertionError
____________ TestUltraJSONTests.test_invalid_double_precision[None] ____________

self = <pandas.tests.io.json.test_ujson.TestUltraJSONTests object at 0x7fe5ee07d990>
invalid_val = None

@pytest.mark.parametrize("invalid_val", [20, -1, "9", None])
def test_invalid_double_precision(self, invalid_val):
    double_input = 30.12345678901234567890
    expected_exception = ValueError if isinstance(invalid_val, int) else TypeError
    msg = (
        r"Invalid value '.*' for option 'double_precision', max is '15'|"
        r"an integer is required \(got type "
    )
    with pytest.raises(expected_exception, match=msg):

      ujson.encode(double_input, double_precision=invalid_val)

E TypeError: 'NoneType' object cannot be interpreted as an integer

pandas/tests/io/json/test_ujson.py:260: TypeError

During handling of the above exception, another exception occurred:

self = <pandas.tests.io.json.test_ujson.TestUltraJSONTests object at 0x7fe5ee07d990>
invalid_val = None

@pytest.mark.parametrize("invalid_val", [20, -1, "9", None])
def test_invalid_double_precision(self, invalid_val):
    double_input = 30.12345678901234567890
    expected_exception = ValueError if isinstance(invalid_val, int) else TypeError
    msg = (
        r"Invalid value '.*' for option 'double_precision', max is '15'|"
        r"an integer is required \(got type "
    )

  with pytest.raises(expected_exception, match=msg):

E AssertionError: Regex pattern "Invalid value '.*' for option 'double_precision', max is '15'|an integer is required \(got type " does not match "'NoneType' object cannot be interpreted as an integer".

pandas/tests/io/json/test_ujson.py:259: AssertionError

• TestGetDummies.test_get_dummies_basic_drop_first_NA

__________ TestGetDummies.test_get_dummies_basic_drop_first_NA[dense] __________

self = <pandas.tests.reshape.test_get_dummies.TestGetDummies object at 0x7fe5e7b1d7b0>
sparse = False

def test_get_dummies_basic_drop_first_NA(self, sparse):
    # Test NA handling together with drop_first
    s_NA = ["a", "b", np.nan]
    res = get_dummies(s_NA, drop_first=True, sparse=sparse)
    exp = DataFrame({"b": [0, 1, 0]}, dtype=np.uint8)
    if sparse:
        exp = exp.apply(SparseArray, fill_value=0)

    tm.assert_frame_equal(res, exp)

    res_na = get_dummies(s_NA, dummy_na=True, drop_first=True, sparse=sparse)
    exp_na = DataFrame({"b": [0, 1, 0], np.nan: [0, 0, 1]}, dtype=np.uint8).reindex(
        ["b", np.nan], axis=1
    )
    if sparse:
        exp_na = exp_na.apply(SparseArray, fill_value=0)

  tm.assert_frame_equal(res_na, exp_na)

E AssertionError

pandas/tests/reshape/test_get_dummies.py:485: AssertionError

• test_reindex_nan

_______________________________ test_reindex_nan _______________________________

def test_reindex_nan():
    ts = Series([2, 3, 5, 7], index=[1, 4, np.nan, 8])

    i, j = [np.nan, 1, np.nan, 8, 4, np.nan], [2, 0, 2, 3, 1, 2]
    tm.assert_series_equal(ts.reindex(i), ts.iloc[j])

    ts.index = ts.index.astype("object")

    # reindex coerces index.dtype to float, loc/iloc doesn't

  tm.assert_series_equal(ts.reindex(i), ts.iloc[j], check_index_type=False)

E AssertionError: Attributes of Series are different
E
E Attribute "dtype" are different
E [left]: float64
E [right]: int64

• test_map_missing_mixed

_________________ test_map_missing_mixed[vals0-mapping0-exp0] __________________ [gw1] linux -- Python 3.10.0 /opt/hostedtoolcache/Python/3.10.0-beta.2/x64/bin/python

vals = ['a', 'b', 'c'], mapping = {nan: 'not NaN'}
exp = [nan, nan, nan, 'not NaN']

@pytest.mark.parametrize(
    "vals,mapping,exp",
    [
        (list("abc"), {np.nan: "not NaN"}, [np.nan] * 3 + ["not NaN"]),
        (list("abc"), {"a": "a letter"}, ["a letter"] + [np.nan] * 3),
        (list(range(3)), {0: 42}, [42] + [np.nan] * 3),
    ],
)
def test_map_missing_mixed(vals, mapping, exp):
    # GH20495
    s = Series(vals + [np.nan])
    result = s.map(mapping)

  tm.assert_series_equal(result, Series(exp))

pandas/tests/apply/test_series_apply.py:821:

---

pandas/_libs/testing.pyx:53: in pandas._libs.testing.assert_almost_equal
cpdef assert_almost_equal(a, b,

---

raise_assert_detail(obj, msg, lobj, robj, index_values=index_values)
E AssertionError: Series are different
E
E Series values are different (25.0 %)
E [index]: [0, 1, 2, 3]
E [left]: [nan, nan, nan, nan]
E [right]: [nan, nan, nan, not NaN]

pandas/_libs/testing.pyx:168: AssertionError

NOTE: This list is not complete since there are also some failing JSON tests that freeze pytest. Will look at those soon.

[simonjayhawkins]

@lithomas1 when we added support for 3.9, we did it in a patch release. So if we need to make changes, we'll probably backport them, so we probably want ci changes on 1.3.x also.

[lithomas1]

@simonjayhawkins Sure, moved right now. I'm just milestoning so we don't forget.

[jbrockmendel]

@lithomas1 are you able to reproduce the numpy failures locally xref #41935?

[jreback]

@lithomas1 can you update this after merging the patch.

[realead]

I cannot test it easily, but my guess is that #41988 had fixed the following examples:

TestUnique.test_first_nan_kept, TestIsin.test_different_nans, TestIsin.test_different_nan_objects, TestDataFrameSelectReindex.test_reindex_nan, TestIndex.test_isin_nan_common_object, TestGetDummies.test_get_dummies_basic_drop_first_NA, test_map_missing_mixed, test_reindex_nan

test_hash_equal isn't fixed but has the same reason for the failure as the above: python/cpython@a07da09.

[lithomas1]

I'll test w/ CI in #41595, but having some difficulties due to #41935.