Merge pull request #64 from python-hyper/headers-are-bytes

Always return headers as bytes

Author: njsmith

docs/source/changes.rst

@@ -3,6 +3,15 @@ History of changes
 
 .. currentmodule:: h11
 
+vNEXT (unreleased)
+------------------
+
+Bug fixes:
+
+* Always return headers as ``bytes`` objects (`#60
+  <https://github.com/python-hyper/h11/issues/60>`__)
+
+
 v0.8.0 (2018-03-20)
 -------------------
 

h11/_readers.py

@@ -51,7 +51,11 @@ def _obsolete_line_fold(lines):
 
 def _decode_header_lines(lines):
     for line in _obsolete_line_fold(lines):
-        matches = validate(header_field_re, line)
+        # _obsolete_line_fold yields either bytearray or bytes objects. On
+        # Python 3, validate() takes either and returns matches as bytes. But
+        # on Python 2, validate can return matches as bytearrays, so we have
+        # to explicitly cast back.
+        matches = validate(header_field_re, bytes(line))
         yield (matches["field_name"], matches["field_value"])
 
 request_line_re = re.compile(request_line.encode("ascii"))

h11/tests/test_io.py

@@ -58,22 +58,31 @@ def makebuf(data):
     return buf
 
 def tr(reader, data, expected):
+    def check(got):
+        assert got == expected
+        # Headers should always be returned as bytes, not e.g. bytearray
+        # https://github.com/python-hyper/wsproto/pull/54#issuecomment-377709478
+        for name, value in getattr(got, "headers", []):
+            print(name, value)
+            assert type(name) is bytes
+            assert type(value) is bytes
+
     # Simple: consume whole thing
     buf = makebuf(data)
-    assert reader(buf) == expected
+    check(reader(buf))
     assert not buf
 
     # Incrementally growing buffer
     buf = ReceiveBuffer()
     for i in range(len(data)):
         assert reader(buf) is None
         buf += data[i:i + 1]
-    assert reader(buf) == expected
+    check(reader(buf))
 
-    # Extra
+    # Trailing data
     buf = makebuf(data)
     buf += b"trailing"
-    assert reader(buf) == expected
+    check(reader(buf))
     assert bytes(buf) == b"trailing"
 
 def test_writers_simple():
@@ -157,12 +166,15 @@ def test_readers_unusual():
        b" header\r\n"
        b"\tnonsense\r\n"
        b"    \t   \t\tI guess\r\n"
-       b"Connection: close\r\n\r\n",
+       b"Connection: close\r\n"
+       b"More-nonsense: in the\r\n"
+       b"    last header  \r\n\r\n",
        Request(method="HEAD", target="/foo",
                headers=[
                    ("Host", "example.com"),
                    ("Some", "multi-line header nonsense I guess"),
                    ("Connection", "close"),
+                   ("More-nonsense", "in the last header"),
                    ]))
 
     with pytest.raises(LocalProtocolError):