Skip to content

Add multi-dimensional arrays support for existing elementwise tests #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 23 commits into from
Oct 7, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions array_api_tests/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
from hypothesis.extra.array_api import make_strategies_namespace

from . import _array_module as xp


xps = make_strategies_namespace(xp)
35 changes: 15 additions & 20 deletions array_api_tests/hypothesis_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from hypothesis.strategies import (lists, integers, sampled_from,
shared, floats, just, composite, one_of,
none, booleans)
from hypothesis.extra.array_api import make_strategies_namespace

from .pytest_helpers import nargs
from .array_helpers import (dtype_ranges, integer_dtype_objects,
Expand All @@ -15,15 +14,12 @@
integer_or_boolean_dtype_objects, dtype_objects)
from ._array_module import (full, float32, float64, bool as bool_dtype,
_UndefinedStub, eye, broadcast_to)
from . import _array_module
from . import _array_module as xp
from . import xps

from .function_stubs import elementwise_functions


xps = make_strategies_namespace(xp)


# Set this to True to not fail tests just because a dtype isn't implemented.
# If no compatible dtype is implemented for a given test, the test will fail
# with a hypothesis health check error. Note that this functionality will not
Expand Down Expand Up @@ -77,10 +73,6 @@ def mutually_promotable_dtypes(draw, dtype_objects=dtype_objects):
dtype_pairs = [(i, j) for i, j in dtype_pairs if i in dtype_objects and j in dtype_objects]
return draw(sampled_from(dtype_pairs))

shared_mutually_promotable_dtype_pairs = shared(
mutually_promotable_dtypes(), key="mutually_promotable_dtype_pair"
)

# shared() allows us to draw either the function or the function name and they
# will both correspond to the same function.

Expand All @@ -91,10 +83,10 @@ def mutually_promotable_dtypes(draw, dtype_objects=dtype_objects):
lambda func_name: nargs(func_name) > 1)

elementwise_function_objects = elementwise_functions_names.map(
lambda i: getattr(_array_module, i))
lambda i: getattr(xp, i))
array_functions = elementwise_function_objects
multiarg_array_functions = multiarg_array_functions_names.map(
lambda i: getattr(_array_module, i))
lambda i: getattr(xp, i))

# Limit the total size of an array shape
MAX_ARRAY_SIZE = 10000
Expand Down Expand Up @@ -164,7 +156,6 @@ def two_broadcastable_shapes(draw):
sizes = integers(0, MAX_ARRAY_SIZE)
sqrt_sizes = integers(0, SQRT_MAX_ARRAY_SIZE)

# TODO: Generate general arrays here, rather than just scalars.
numeric_arrays = xps.arrays(
dtype=shared(xps.floating_dtypes(), key='dtypes'),
shape=shared(xps.array_shapes(), key='shapes'),
Expand Down Expand Up @@ -275,14 +266,18 @@ def multiaxis_indices(draw, shapes):
return tuple(res)


shared_arrays1 = xps.arrays(
dtype=shared_mutually_promotable_dtype_pairs.map(lambda pair: pair[0]),
shape=shared(two_mutually_broadcastable_shapes, key="shape_pair").map(lambda pair: pair[0]),
)
shared_arrays2 = xps.arrays(
dtype=shared_mutually_promotable_dtype_pairs.map(lambda pair: pair[1]),
shape=shared(two_mutually_broadcastable_shapes, key="shape_pair").map(lambda pair: pair[1]),
)
def two_mutual_arrays(dtypes=dtype_objects):
mutual_dtypes = shared(mutually_promotable_dtypes(dtypes))
mutual_shapes = shared(two_mutually_broadcastable_shapes)
arrays1 = xps.arrays(
dtype=mutual_dtypes.map(lambda pair: pair[0]),
shape=mutual_shapes.map(lambda pair: pair[0]),
)
arrays2 = xps.arrays(
dtype=mutual_dtypes.map(lambda pair: pair[1]),
shape=mutual_shapes.map(lambda pair: pair[1]),
)
return arrays1, arrays2


@composite
Expand Down
14 changes: 10 additions & 4 deletions array_api_tests/meta_tests/test_hypothesis_helpers.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
from math import prod

import pytest
from hypothesis import given, strategies as st
from hypothesis import given, strategies as st, settings

from .. import _array_module as xp
from .._array_module import _UndefinedStub
from .. import array_helpers as ah
from .. import hypothesis_helpers as hh
from ..test_broadcasting import broadcast_shapes
from ..test_elementwise_functions import sanity_check

UNDEFINED_DTYPES = any(isinstance(d, _UndefinedStub) for d in ah.dtype_objects)
pytestmark = [pytest.mark.skipif(UNDEFINED_DTYPES, reason="undefined dtypes")]
Expand Down Expand Up @@ -44,20 +46,24 @@ def test_two_mutually_broadcastable_shapes(pair):
def test_two_broadcastable_shapes(pair):
for shape in pair:
assert valid_shape(shape)
assert broadcast_shapes(pair[0], pair[1]) == pair[0]

from ..test_broadcasting import broadcast_shapes

assert broadcast_shapes(pair[0], pair[1]) == pair[0]
@given(*hh.two_mutual_arrays())
def test_two_mutual_arrays(x1, x2):
sanity_check(x1, x2)
assert broadcast_shapes(x1.shape, x2.shape) in (x1.shape, x2.shape)


def test_kwargs():
results = []

@given(hh.kwargs(n=st.integers(0, 10), c=st.from_regex("[a-f]")))
@settings(max_examples=100)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the purpose of this? Isn't max_examples already 100 by default? If we set the --max-examples flag at the command line (see conftest.py) which takes precedence?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep @settings takes precedence over everything else. I do this so this test should pass even when we use --max-examples=1. This is actually what a few tests in Hypothesis does, to prevent false-negatives.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand why you would ever pass max-examples=1. And even if you did, why wouldn't want it to apply to this test? I typically use max-examples to increase the number of examples to do a more rigorous test run.

Copy link
Member Author

@honno honno Oct 7, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, a few times now I've run pytest --max-examples=1 for the whole suite, so I thought forcing max_examples here would reduce noise for myself and generally prevent any noisy false-negatives if someone else had the same idea. Generally I think it clarifies that we're not using Hypothesis to test things, but testing Hypothesis itself i.e. see if our custom strategy will work for a "typical" run. I imagine both of these reasons are why Hypothesis does this internally too.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this test fail if it is run with max_examples=1?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, as expected. In Hypothesis this kind of max_example forcing would be covered by their find_any util.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How this test_kwargs test method tries to find that certain things will be generated is essentially what find_any does in their own test suite.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. I missed that. I didn't notice that this was on an inner test function, not the outer one.

def run(kw):
results.append(kw)

run()

assert all(isinstance(kw, dict) for kw in results)
for size in [0, 1, 2]:
assert any(len(kw) == size for kw in results)
Expand Down
3 changes: 2 additions & 1 deletion array_api_tests/test_creation_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
assert_exactly_equal, isintegral, is_float_dtype)
from .hypothesis_helpers import (numeric_dtypes, dtypes, MAX_ARRAY_SIZE,
shapes, sizes, sqrt_sizes, shared_dtypes,
scalars, xps, kwargs)
scalars, kwargs)
from . import xps

from hypothesis import assume, given
from hypothesis.strategies import integers, floats, one_of, none, booleans, just, shared, composite
Expand Down
Loading