Merge pull request #4695 from HypothesisWorks/DRMacIver/syrupy

Add snapshot testing with syrupy

Author: DRMacIver

.github/workflows/main.yml

@@ -278,7 +278,7 @@ jobs:
       run: |
         pip install --upgrade setuptools pip wheel build
         python -m build --wheel hypothesis-python --outdir dist/
-        pip download --dest=dist/ hypothesis-python/ pytest tzdata  # fetch all the wheels
+        pip download --dest=dist/ hypothesis-python/ pytest tzdata syrupy  # fetch all the wheels
          rm dist/packaging-*.whl  # fails with `invalid metadata entry 'name'`
          pyodide venv .venv-pyodide
         source .venv-pyodide/bin/activate

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+This release improves |Phase.explain| output by excluding pytest-related :pypi:`syrupy` files as a possible source of variation.

hypothesis-python/src/hypothesis/internal/scrutineer.py

@@ -167,6 +167,9 @@ def __exit__(self, *args, **kwargs):
     "/typing.py",
     "/conftest.py",
     "/pprint.py",
+    # syrupy registers a pytest_assertrepr_compare hook, which only runs when
+    # assertions fail — making it appear as always-failing-never-passing.
+    "/syrupy/__init__.py",
 )
 
 

hypothesis-python/tests/common/utils.py

@@ -333,3 +333,9 @@ def wait_for(condition, *, timeout=1, interval=0.01):
         f"timing out after waiting {timeout}s for condition "
         f"{get_pretty_function_description(condition)}"
     )
+
+
+def run_test_for_falsifying_example(test_fn):
+    with pytest.raises(AssertionError) as err:
+        test_fn()
+    return "\n".join(err.value.__notes__).strip()

hypothesis-python/tests/cover/__snapshots__/test_custom_reprs.ambr

@@ -0,0 +1,44 @@
+# serializer version: 1
+# name: test_map_to_bytes_prints_as_repr
+  '''
+  Falsifying example: inner(
+      b=b"\xe3\xb0\xc4B\x98\xfc\x1c\x14\x9a\xfb\xf4\xc8\x99o\xb9$'\xaeA\xe4d\x9b\x93L\xa4\x95\x99\x1bxR\xb8U",
+  )
+  '''
+# ---
+# name: test_map_to_str_prints_as_repr
+  '''
+  Falsifying example: inner(
+      s='0',
+  )
+  '''
+# ---
+# name: test_reprs_as_created
+  '''
+  Falsifying example: inner(
+      foo=Foo(x=1),
+      bar=Bar(x=-1),
+      baz=Foo(None),
+  )
+  '''
+# ---
+# name: test_reprs_as_created_consistent_calls_despite_indentation
+  '''
+  Falsifying example: inner(
+      a=some_foo('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'),
+      b=Bar(
+          some_foo(
+              'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+          ),
+      ),
+  )
+  '''
+# ---
+# name: test_reprs_as_created_interactive
+  '''
+  Falsifying example: inner(
+      data=data(...),
+  )
+  Draw 1: Bar(10)
+  '''
+# ---

hypothesis-python/tests/cover/test_custom_reprs.py

@@ -17,6 +17,12 @@
 from hypothesis.strategies._internal.lazy import unwrap_strategies
 
 
+def _get_output(test_fn):
+    with pytest.raises(AssertionError) as err:
+        test_fn()
+    return "\n".join(err.value.__notes__).strip()
+
+
 def test_includes_non_default_args_in_repr():
     assert repr(st.integers()) == "integers()"
     assert repr(st.integers(min_value=1)) == "integers(min_value=1)"
@@ -79,41 +85,24 @@ class Bar(Foo):
     pass
 
 
-def test_reprs_as_created():
+def test_reprs_as_created(snapshot):
     @given(foo=st.builds(Foo), bar=st.from_type(Bar), baz=st.none().map(Foo))
     @settings(print_blob=False, max_examples=10_000, derandomize=True)
     def inner(foo, bar, baz):
         assert baz.x is None
         assert foo.x <= 0 or bar.x >= 0
 
-    with pytest.raises(AssertionError) as err:
-        inner()
-    expected = """
-Falsifying example: inner(
-    foo=Foo(x=1),
-    bar=Bar(x=-1),
-    baz=Foo(None),
-)
-"""
-    assert "\n".join(err.value.__notes__).strip() == expected.strip()
+    assert _get_output(inner) == snapshot
 
 
-def test_reprs_as_created_interactive():
+def test_reprs_as_created_interactive(snapshot):
     @given(st.data())
     @settings(print_blob=False, max_examples=10_000)
     def inner(data):
         bar = data.draw(st.builds(Bar, st.just(10)))
         assert not bar.x
 
-    with pytest.raises(AssertionError) as err:
-        inner()
-    expected = """
-Falsifying example: inner(
-    data=data(...),
-)
-Draw 1: Bar(10)
-"""
-    assert "\n".join(err.value.__notes__).strip() == expected.strip()
+    assert _get_output(inner) == snapshot
 
 
 CONSTANT_FOO = Foo(None)
@@ -143,7 +132,7 @@ def inner(a, b):
     assert re.fullmatch(expected_re, got), got
 
 
-def test_reprs_as_created_consistent_calls_despite_indentation():
+def test_reprs_as_created_consistent_calls_despite_indentation(snapshot):
     aas = "a" * 60
     strat = st.builds(some_foo, st.just(aas))
 
@@ -154,19 +143,7 @@ def test_reprs_as_created_consistent_calls_despite_indentation():
     def inner(a, b):
         assert a == b
 
-    with pytest.raises(AssertionError) as err:
-        inner()
-    expected = f"""
-Falsifying example: inner(
-    a=some_foo({aas!r}),
-    b=Bar(
-        some_foo(
-            {aas!r},
-        ),
-    ),
-)
-"""
-    assert "\n".join(err.value.__notes__).strip() == expected.strip()
+    assert _get_output(inner) == snapshot
 
 
 @pytest.mark.parametrize(
@@ -199,33 +176,19 @@ def test_characters_repr(strategy, expected_repr):
     assert repr(unwrap_strategies(strategy)) == expected_repr
 
 
-def test_map_to_str_prints_as_repr():
+def test_map_to_str_prints_as_repr(snapshot):
     @given(s=st.integers().map(str))
     @settings(phases=[Phase.generate, Phase.shrink], print_blob=False)
     def inner(s):
         raise AssertionError
 
-    with pytest.raises(AssertionError) as err:
-        inner()
-    expected = """
-Falsifying example: inner(
-    s='0',
-)
-"""
-    assert "\n".join(err.value.__notes__).strip() == expected.strip()
+    assert _get_output(inner) == snapshot
 
 
-def test_map_to_bytes_prints_as_repr():
+def test_map_to_bytes_prints_as_repr(snapshot):
     @given(b=st.binary().map(lambda b: hashlib.sha256(b).digest()))
     @settings(phases=[Phase.generate, Phase.shrink], print_blob=False)
     def inner(b):
         raise AssertionError
 
-    with pytest.raises(AssertionError) as err:
-        inner()
-    expected = f"""
-Falsifying example: inner(
-    b={hashlib.sha256(b"").digest()!r},
-)
-"""
-    assert "\n".join(err.value.__notes__).strip() == expected.strip()
+    assert _get_output(inner) == snapshot

hypothesis-python/tests/snapshots/__init__.py

@@ -0,0 +1,9 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Copyright the Hypothesis Authors.
+# Individual contributors are listed in AUTHORS.rst and the git log.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.