Add procedure for determining token values for tokens of types that do not feature a value attribute

Tokenization as defined in the specification effectively maps code points in the input text stream, to certain token types where the "value" is implied -- a comma token, for instance, is created when the `,` is encountered in the input stream. We envision needing the inverse function which returns the code point given a token's type. This commit adds an implementation of such function.

Author: amn

expand/csspring/syntax/tokenizing.py

@@ -9,6 +9,7 @@
 from ..utils import CP, BufferedPeekingReader, is_surrogate_code_point_ordinal, IteratorReader, join, parser_error, PeekingUnreadingReader
 
 from abc import ABC
+import builtins
 from collections.abc import Callable, Iterable, Iterator
 from dataclasses import dataclass
 from decimal import Decimal
@@ -502,3 +503,26 @@ def is_non_printable_code_point(cp: CP) -> bool:
 def is_whitespace(cp: CP) -> bool:
     """See http://drafts.csswg.org/css-syntax/#whitespace."""
     return is_newline(cp) or cp in ('\t', ' ')
+
+# Map of values by token type, for types of tokens which do _not_ have the `value` attribute
+token_values = { # For the `token_value` procedure to work as intended, subtypes should be listed _before_ their supertype(s)
+    OpenBraceToken: '{',
+    OpenBracketToken: '[',
+    OpenParenToken: '(',
+    CloseBraceToken: '}',
+    CloseBracketToken: ']',
+    CloseParenToken: ')',
+    ColonToken: ':',
+    CommaToken: ',',
+    SemicolonToken: ';',
+    CDCToken: '->',
+    CDOToken: '!--',
+}
+
+def token_value(type: builtins.type[Token]) -> str:
+    """Get the value of a token by its type for types of tokens that do _not_ feature a `value` attribute.
+
+    :param type: Type of token to get the value of
+    :returns: The value common to the type of tokens
+    """
+    return next(value for key, value in token_values.items() if issubclass(type, key))