Fix issue with header prefixes, remove ambiguous header processing, add test

Author: zootalures

README.md

@@ -1,11 +1,55 @@
 # Function development kit for Python
+The python FDK lets you write functions in python 3.6/3.7 
 
+## Simplest possible function 
+ 
+```python
+import io
+import logging
+
+from fdk import response
+
+def handler(ctx, data: io.BytesIO = None):
+    logging.getLogger().info("Got incoming request")
+    return response.Response(ctx, response_data="hello world")
+```
 
-While the FDK contract is HTTP, the intention is for that to be somewhat 
-abstracted from the user - they write some Function code , this library helps them do that.
 
-## Handling JSON Functions
+## Handling HTTP metadata in HTTP Functions 
+Functions can implement HTTP services when fronted by an HTTP Gateway
 
+When your function is behind an HTTP gateway you can access the inbound HTTP Request via :
+
+  - `ctx.HttpHeaders()` : a map of string -> value | list of values , unlike `ctx.Headers()` this only includes headers 
+        passed by the HTTP gateway (with no functions metadata).
+  - `ctx.RequestURL()` : the incoming request URL passed by the gateway 
+  - `ctx.Method()` : the HTTP method of the incoming request 
+   
+You can set outbound HTTP headers and the HTTP status of the request using `ctx.SetResponseHeaders` or the `Response`    
+  - e.g. `ctx.SetResponseHeaders({"Location","http://example.com/","My-Header2": ["v1","v2"]}, 302)` 
+  - or by passing these to the Response object : 
+```python
+        return new Response(ctx,
+                        headers={"Location","http://example.com/","My-Header2": ["v1","v2"]},
+                        response_data="Page moved",
+                        status_code=302)
+``` 
+
+e.g. to redirect users to a different page : 
+```python
+import io
+import logging
+
+from fdk import response
+
+def handler(ctx, data: io.BytesIO = None):
+    logging.getLogger().info("Got incoming request for URL %s with headers %s", ctx.RequestURL(), ctx.HTTPHeaders())
+    ctx.SetResponseHeaders({"Location": "http://www.example.com"}, 302)
+    return response.Response(ctx, response_data="Page moved from %s")
+```
+
+
+## Handling JSON in  Functions
 
 A main loop is supplied that can repeatedly call a user function with a series of requests.
 In order to utilise this, you can write your `func.py` as follows:
@@ -16,7 +60,6 @@ import io
 
 from fdk import response
 
-
 def handler(ctx, data: io.BytesIO=None):
     name = "World"
     try:
@@ -34,8 +77,8 @@ def handler(ctx, data: io.BytesIO=None):
 
 ```
 
-## Unittest your functions
 
+## Unit testing your functions
 
 Starting v0.0.33 FDK-Python provides a testing framework that allows performing unit tests of your function's code.
 The unit test framework is the [pytest](https://pytest.org/). Coding style remain the same, so, write your tests as you've got used to.

fdk/context.py

@@ -57,6 +57,7 @@ def __init__(self, app_id, fn_id, call_id,
         self.__call_id = call_id
         self.__config = config if config else {}
         self.__headers = headers if headers else {}
+        self.__http_headers = {}
         self.__deadline = deadline
         self.__content_type = content_type
         self._request_url = request_url
@@ -66,8 +67,10 @@ def __init__(self, app_id, fn_id, call_id,
 
         log.log("request headers. gateway: {0} {1}"
                 .format(self.__is_gateway(), headers))
+
         if self.__is_gateway():
-            self.__headers = hs.decap_headers(self.__headers)
+            self.__headers = hs.decap_headers(headers, True)
+            self.__http_headers = hs.decap_headers(headers, False)
 
     def AppID(self):
         return self.__app_id
@@ -84,6 +87,9 @@ def Config(self):
     def Headers(self):
         return self.__headers
 
+    def HTTPHeaders(self):
+        return self.__http_headers
+
     def Format(self):
         return self.__fn_format
 

fdk/fixtures.py

@@ -24,7 +24,6 @@ async def process_response(fn_call_coro):
     response_data = resp.body()
     response_status = resp.status()
     response_headers = resp.context().GetResponseHeaders()
-    print(response_headers)
 
     return response_data, response_status, response_headers
 
@@ -83,10 +82,17 @@ async def setup_fn_call(
         method=method, request_url=request_url,
         gateway=gateway
     )
+    return await setup_fn_call_raw(handle_func, content, new_headers)
+
+
+async def setup_fn_call_raw(handle_func, content=None, headers=None):
+
+    if headers is None:
+        headers = {}
 
     # don't decap headers, so we can test them
     # (just like they come out of fdk)
     return process_response(runner.handle_request(
         code(handle_func), constants.HTTPSTREAM,
-        headers=new_headers, data=content,
+        headers=headers, data=content,
     ))

fdk/headers.py

@@ -15,29 +15,58 @@
 from fdk import constants
 
 
-def decap_headers(hdsr):
+def decap_headers(hdsr, merge=True):
     ctx_headers = {}
     if hdsr is not None:
         for k, v in hdsr.items():
             k = k.lower()
             if k.startswith(constants.FN_HTTP_PREFIX):
-                ctx_headers[k.lstrip(constants.FN_HTTP_PREFIX)] = v
-            else:
-                ctx_headers[k] = v
+                push_header(ctx_headers, k[len(constants.FN_HTTP_PREFIX):], v)
+            elif merge:
+                # http headers override functions headers in context
+                # this is not ideal but it's the more correct view from the
+                # consumer perspective than random choice and for things
+                # like host headers
+                if k not in ctx_headers:
+                    ctx_headers[k] = v
     return ctx_headers
 
 
+def push_header(input_map, key, value):
+    if key not in input_map:
+        input_map[key] = value
+        return
+
+    current_val = input_map[key]
+
+    if isinstance(current_val, list):
+        if isinstance(value, list):  # both lists concat
+            input_map[key] = current_val + value
+        else:  # copy and append current value
+            new_val = current_val.copy()
+            new_val.append(value)
+            input_map[key] = new_val
+    else:
+        if isinstance(value, list):  # copy new list value and prepend current
+            new_value = value.copy()
+            new_value.insert(0, current_val)
+            input_map[key] = new_value
+        else:  # both non-lists create a new list
+            input_map[key] = [current_val, value]
+
+
 def encap_headers(headers, status=None):
     new_headers = {}
     if headers is not None:
         for k, v in headers.items():
             k = k.lower()
+            if k.startswith(constants.FN_HTTP_PREFIX):  # by default merge
+                push_header(new_headers, k, v)
             if (k == constants.CONTENT_TYPE or
-                    k == constants.FN_FDK_VERSION or
-                    k.startswith(constants.FN_HTTP_PREFIX)):
+                    k == constants.FN_FDK_VERSION):  # but don't merge these
                 new_headers[k] = v
             else:
-                new_headers[constants.FN_HTTP_PREFIX + k] = v
+                push_header(new_headers, constants.FN_HTTP_PREFIX + k, v)
 
     if status is not None:
         new_headers[constants.FN_HTTP_STATUS] = str(status)

fdk/tests/funcs.py

@@ -17,7 +17,6 @@
 
 from fdk import response
 
-
 xml = """<!DOCTYPE mensaje SYSTEM "record.dtd">
 <record>
     <player_birthday>1979-09-23</player_birthday>
@@ -78,7 +77,6 @@ def none_func(ctx, data=None):
 
 
 def timed_sleepr(timeout):
-
     def sleeper(ctx, data=None):
         time.sleep(timeout)
 
@@ -122,3 +120,24 @@ def access_request_url(ctx, **kwargs):
             "Request-Method": method,
         }
     )
+
+
+captured_context = None
+
+
+def setup_context_capture():
+    global captured_context
+    captured_context = None
+
+
+def get_captured_context():
+    global captured_context
+    my_context = captured_context
+    captured_context = None
+    return my_context
+
+
+def capture_request_ctx(ctx, **kwargs):
+    global captured_context
+    captured_context = ctx
+    return response.Response(ctx, response_data="OK")

fdk/tests/test_headers.py

@@ -0,0 +1,109 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from fdk import headers
+
+
+def test_push_header():
+    cases = [
+        ({}, "k", "v", {"k": "v"}),
+        ({}, "k", ["v1", "v2"], {"k": ["v1", "v2"]}),
+        ({"k": "v1"}, "k", "v2", {"k": ["v1", "v2"]}),
+        ({"k": ["v1"]}, "k", "v2", {"k": ["v1", "v2"]}),
+        ({"k": ["v1"]}, "k", ["v2"], {"k": ["v1", "v2"]}),
+        ({"k": []}, "k", [], {"k": []}),
+        ({"k": ["v1"]}, "k", [], {"k": ["v1"]}),
+        ({"k": []}, "k", ["v1"], {"k": ["v1"]}),
+        ({"k": "v1"}, "k", ["v2", "v3"], {"k": ["v1", "v2", "v3"]}),
+        ({"k1": "v1"}, "k2", "v2", {"k1": "v1", "k2": "v2"}),
+
+    ]
+
+    for case in cases:
+        initial = case[0]
+        working = initial.copy()
+        key = case[1]
+        value = case[2]
+        result = case[3]
+        headers.push_header(working, key, value)
+        assert working == result, "Adding  %s:%s to %s" \
+                                  % (key, value, initial)
+
+
+def test_encap_no_headers():
+    encap = headers.encap_headers({})
+    assert not encap, "headers should be empty"
+
+
+def test_encap_simple_headers():
+    encap = headers.encap_headers({
+        "Test-header": "foo",
+        "name-Conflict": "h1",
+        "name-conflict": "h2",
+        "nAme-conflict": ["h3", "h4"],
+        "fn-http-h-name-conflict": "h5",
+        "multi-header": ["bar", "baz"]
+    })
+    assert "fn-http-h-test-header" in encap
+    assert "fn-http-h-name-conflict" in encap
+    assert "fn-http-h-multi-header" in encap
+
+    assert encap["fn-http-h-test-header"] == "foo"
+    assert set(encap["fn-http-h-name-conflict"]) == {"h1", "h2",
+                                                     "h3", "h4", "h5"}
+    assert encap["fn-http-h-multi-header"] == ["bar", "baz"]
+
+
+def test_encap_status():
+    encap = headers.encap_headers({}, 202)
+    assert "fn-http-status" in encap
+    assert encap["fn-http-status"] == "202"
+
+
+def test_encap_status_override():
+    encap = headers.encap_headers({"fn-http-status": 412}, 202)
+    assert "fn-http-status" in encap
+    assert encap["fn-http-status"] == "202"
+
+
+def test_content_type_version():
+    encap = headers.encap_headers({"content-type": "text/plain",
+                                   "fn-fdk-version": "1.2.3"})
+
+    assert encap == {"content-type": "text/plain", "fn-fdk-version": "1.2.3"}
+
+
+def test_decap_headers_merge():
+    decap = headers.decap_headers({"fn-http-h-Foo-Header": "v1",
+                                   "fn-http-h-merge-header": "v2",
+                                   "fn-http-h-merge-Header": ["v3"],
+                                   "Foo-Header": "ignored",
+                                   "other-header": "bob"}, True)
+    assert "foo-header" in decap
+    assert decap["foo-header"] == "v1"
+
+    assert "other-header" in decap
+    assert decap["other-header"] == "bob"
+
+    assert "merge-header" in decap
+    assert set(decap["merge-header"]) == {"v2", "v3"}
+
+
+def test_decap_headers_strip():
+    decap = headers.decap_headers({"fn-http-h-Foo-Header": "v1",
+                                   "fn-http-h-merge-header": ["v2"],
+                                   "Foo-Header": "ignored",
+                                   "merge-header": "v3",
+                                   "other-header": "bad"}, False)
+    assert decap == {"foo-header": "v1", "merge-header": ["v2"]}