Merge pull request #4698 from Liam-DeVoe/more-constant-strings

Add more constant strings

Author: Liam-DeVoe

hypothesis-python/RELEASE.rst

@@ -0,0 +1,5 @@
+RELEASE_TYPE: patch
+
+This patch fixes our |st.from_regex| type annotations so that ``from_regex(..., alphabet=None)`` is accepted.
+
+This patch also adds unicode line breaks and thai combining vowels to our list of constant strings to upweight at runtime.

hypothesis-python/docs/prolog.rst

@@ -104,6 +104,7 @@
 .. |st.ip_addresses| replace:: :func:`~hypothesis.strategies.ip_addresses`
 .. |st.register_type_strategy| replace:: :func:`~hypothesis.strategies.register_type_strategy`
 .. |st.just| replace:: :func:`~hypothesis.strategies.just`
+.. |st.from_regex| replace:: :func:`~hypothesis.strategies.from_regex`
 .. |st.domains| replace:: :func:`~hypothesis.provisional.domains`
 .. |st.urls| replace:: :func:`~hypothesis.provisional.urls`
 .. |register_random| replace:: :func:`~hypothesis.register_random`

hypothesis-python/src/hypothesis/internal/charmap.py

@@ -310,6 +310,12 @@ def query(
         min_codepoint = 0
     if max_codepoint is None:
         max_codepoint = sys.maxunicode
+
+    if min_codepoint > max_codepoint:
+        raise InvalidArgument(
+            f"min_codepoint={min_codepoint} is greater than max_codepoint={max_codepoint}"
+        )
+
     catkey = _category_key(categories)
     character_intervals = IntervalSet.from_string("".join(include_characters))
     exclude_intervals = IntervalSet.from_string("".join(exclude_characters))
@@ -326,12 +332,14 @@ def query(
             return limited_category_index_cache[qkey]
         except KeyError:
             pass
-    base = _query_for_key(catkey)
+
     result = []
-    for u, v in base:
+    for u, v in _query_for_key(catkey):
         if v >= min_codepoint and u <= max_codepoint:
             result.append((max(u, min_codepoint), min(v, max_codepoint)))
+
     result = (IntervalSet(result) | character_intervals) - exclude_intervals
     if context is None or not context.data.provider.avoid_realization:
         limited_category_index_cache[qkey] = result
+
     return result

hypothesis-python/src/hypothesis/internal/conjecture/providers.py

@@ -230,6 +230,9 @@
     # Ogham text, which contains the only character in the Space Separators
     # unicode category (Zs) that isn't visually blank:  .  # noqa: RUF003
     "᚛ᚄᚓᚐᚋᚒᚄ ᚑᚄᚂᚑᚏᚅ᚜",
+    # thai consonant + spacing vowel combinations, which have unusual visual combining behavior
+    "กา",
+    "ก ำกำ",
     # readable variations on text (bolt/italic/script)
     "𝐓𝐡𝐞 𝐪𝐮𝐢𝐜𝐤 𝐛𝐫𝐨𝐰𝐧 𝐟𝐨𝐱 𝐣𝐮𝐦𝐩𝐬 𝐨𝐯𝐞𝐫 𝐭𝐡𝐞 𝐥𝐚𝐳𝐲 𝐝𝐨𝐠",
     "𝕿𝖍𝖊 𝖖𝖚𝖎𝖈𝖐 𝖇𝖗𝖔𝖜𝖓 𝖋𝖔𝖝 𝖏𝖚𝖒𝖕𝖘 𝖔𝖛𝖊𝖗 𝖙𝖍𝖊 𝖑𝖆𝖟𝖞 𝖉𝖔𝖌",
@@ -251,6 +254,33 @@
     "मनीष منش",
     "पन्ह पन्ह त्र र्च कृकृ ड्ड न्हृे إلا بسم الله",
     "lorem لا بسم الله ipsum 你好1234你好",
+    # unicode charaacters causing unconditional line breaks, as defined by UAX #14:
+    # https://www.unicode.org/reports/tr14/.
+    #
+    # We've seen multiple bugs caused by assuming `str.splitlines` is equivalent to
+    # splitting over "\n", while it actually splits over all line breaks!
+    #
+    # We intersperse the line breaks with normal characters to increase the likelihood
+    # of triggering such a bug.
+    (
+        "a"
+        "\u000a"  # line feed (class: LF)
+        "b"
+        "\u000d"  # carriage return (class: CR)
+        "c"
+        "\u0085"  # next line (class: NL)
+        "d"
+        "\u000b"  # line tabulation (class: BK)
+        "e"
+        "\u000c"  # form feed (class: BK)
+        "f"
+        "\u2028"  # line separator (class: BK)
+        "g"
+        "\u2029"  # paragraph separator (class: BK)
+        "h"
+        "\u000d\u000a"  # CR+LF
+        "i"
+    ),
 }
 
 

hypothesis-python/src/hypothesis/strategies/_internal/core.py

@@ -877,7 +877,7 @@ def from_regex(
     regex: str | Pattern[str],
     *,
     fullmatch: bool = False,
-    alphabet: str | SearchStrategy[str] = characters(codec="utf-8"),
+    alphabet: str | SearchStrategy[str] | None = characters(codec="utf-8"),
 ) -> SearchStrategy[str]:  # pragma: no cover
     ...
 

hypothesis-python/tests/cover/test_charmap.py

@@ -18,6 +18,7 @@
 import pytest
 
 from hypothesis import given, strategies as st
+from hypothesis.errors import InvalidArgument
 from hypothesis.internal import charmap as cm
 from hypothesis.internal.intervalsets import IntervalSet
 
@@ -191,6 +192,11 @@ def test_regenerate_broken_charmap_file():
     cm.charmap()
 
 
+def test_query_rejects_min_codepoint_greater_than_max():
+    with pytest.raises(InvalidArgument):
+        cm.query(min_codepoint=1, max_codepoint=0)
+
+
 def test_exclude_characters_are_included_in_key():
     assert cm.query().intervals != cm.query(exclude_characters="0").intervals
 

hypothesis-python/tests/cover/test_intervalset.py

@@ -23,6 +23,7 @@
 
 
 @given(intervals(max_codepoint=200))
+@settings(deadline=None)
 def test_intervals_are_equivalent_to_their_lists(intervals):
     ls = list(intervals)
     assert len(ls) == len(intervals)

whole_repo_tests/types/revealed_types.py

@@ -62,6 +62,7 @@
         "one_of(integers(), text(), none(), binary(), builds(list), builds(dict))",
         "Any",
     ),
+    ("from_regex(r'.', alphabet=None)", "str"),
 ]