added tests for RetryExecutor

Author: igorcoding

.gitlab-ci.yml

@@ -1,3 +1,3 @@
-include:
-  project: mnt/ci
-  file: modules/python.yaml
+#include:
+#  project: mnt/ci
+#  file: modules/python.yaml

extapi/http/abc.py

@@ -160,7 +160,8 @@ async def need_retry(self, response: Response[T]) -> tuple[bool, float | None]:
 
 @runtime_checkable
 class Addon(Protocol[T]):
-    async def before_request(self, request: RequestData) -> None: ...
+    async def before_request(self, request: RequestData) -> None:
+        return None
 
     async def process_response(
         self, request: RequestData, response: Response[T]

extapi/http/addons/log.py

@@ -37,7 +37,7 @@ async def process_error(self, request: RequestData, error: Exception) -> None:
                 "timeout error for request %s %s failed with error %s(%s)",
                 request.method,
                 str(request.url),
-                type(error),
+                type(error).__name__,
                 str(error),
             )
         elif isinstance(error, HttpExecuteError):
@@ -47,7 +47,7 @@ async def process_error(self, request: RequestData, error: Exception) -> None:
                 "request %s %s failed with error %s(%s)",
                 request.method,
                 str(request.url),
-                type(error),
+                type(error).__name__,
                 error,
             )
 

extapi/http/addons/retry.py

@@ -7,9 +7,14 @@
 
 
 class Retry5xxAddon(Retryable[T], Generic[T]):
+    __slots__ = ("_retry_timeout",)
+
+    def __init__(self, *, retry_timeout: float | None = None):
+        self._retry_timeout = retry_timeout
+
     async def need_retry(self, response: Response[T]) -> tuple[bool, float | None]:
         if response.status >= 500:
-            return True, None
+            return True, self._retry_timeout
 
         return False, None
 

extapi/http/executors/retry.py

@@ -122,7 +122,7 @@ async def execute(self, request: RequestData) -> Response[T]:
                 except Exception as e:
                     self._logger.error(
                         "error post-processing request execution error %s(%s): %s",
-                        type(last_exc),
+                        type(last_exc).__name__,
                         last_exc,
                         e,
                     )
@@ -135,7 +135,7 @@ async def execute(self, request: RequestData) -> Response[T]:
 
         if last_exc is not None:
             raise ExecuteError(
-                f"request failed after {self._max_retries} retries: {str(last_exc)}"
+                f"request failed after {self._max_retries} retries: {type(last_exc).__name__}({str(last_exc)})"
             ) from last_exc
-        else:
+        else:  # pragma: no cover
             raise ExecuteError(f"request failed after {self._max_retries} retries")

tests/exthttp/executors/test_retry.py

@@ -1,3 +1,236 @@
+import inspect
+import time
+from collections.abc import Iterable
+from typing import Any
+
+import pytest
+
+from extapi.http.abc import AbstractExecutor, Addon
+from extapi.http.addons.auth import BearerAuthAddon
+from extapi.http.addons.retry import Retry5xxAddon
+from extapi.http.executors.retry import RetryableExecutor
+from extapi.http.types import ExecuteError, HttpExecuteError, RequestData, Response
+
+
+class _DummyExecutor(AbstractExecutor[Any]):
+    def __init__(self, responses: Iterable[int | type[Exception] | Exception] = (200,)):
+        self.call_count = 0
+        self._responses: list[int | type[Exception] | Exception] = list(responses)
+        self._current_response_index = 0
+
+    async def execute(self, request: RequestData) -> Response:
+        self.call_count += 1
+        if self._current_response_index < len(self._responses):
+            response = self._responses[self._current_response_index]
+            self._current_response_index += 1
+        else:  # pragma: no cover
+            response = self._responses[-1]  # Return the last response if we've run out
+
+        if isinstance(response, Exception) or (
+            inspect.isclass(response) and issubclass(response, Exception)
+        ):
+            raise response
+
+        if isinstance(response, int):
+            return Response(status=response, backend_response="heyo", url=request.url)
+
+        raise BaseException(
+            f"unexpected response type: {type(response)}"
+        )  # pragma: no cover
+
+
 class TestRetryExecutor:
-    def test(self):
-        pass
+    async def test_basic(self, request_simple: RequestData):
+        base = _DummyExecutor()
+        executor = RetryableExecutor(base, retry_sleep_timeout=0)
+
+        response = await executor.execute(request_simple)
+
+        assert response.status == 200
+        assert response.backend_response == "heyo"
+        assert response.url == request_simple.url
+
+    async def test_retry_500_no_addon(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[500, 200])
+        executor = RetryableExecutor(base, max_retries=2, retry_sleep_timeout=0)
+
+        response = await executor.execute(request_simple)
+
+        assert response.status == 500
+        assert base.call_count == 1
+
+    async def test_retry_500_with_addon(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[500, 200])
+        executor = RetryableExecutor(
+            base,
+            max_retries=2,
+            retry_sleep_timeout=0,
+            addons=[
+                Retry5xxAddon(),
+            ],
+        )
+
+        response = await executor.execute(request_simple)
+
+        assert response.status == 200
+        assert base.call_count == 2
+
+    async def test_max_retries_exceeded(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[500, 500, 500])
+        executor = RetryableExecutor(
+            base,
+            max_retries=3,
+            retry_sleep_timeout=0,
+            addons=[
+                Retry5xxAddon(),
+            ],
+        )
+
+        response = await executor.execute(request_simple)
+
+        assert response.status == 500
+        assert base.call_count == 3
+
+    async def test_non_retryable_status(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[400])
+        executor = RetryableExecutor(
+            base,
+            max_retries=2,
+            retry_sleep_timeout=0,
+            addons=[
+                Retry5xxAddon(),
+            ],
+        )
+
+        response = await executor.execute(request_simple)
+
+        assert response.status == 400
+        assert base.call_count == 1
+
+    async def test_retry_with_bearer_auth_error(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[401, 401, 200])
+        executor = RetryableExecutor(
+            base,
+            max_retries=2,
+            retry_sleep_timeout=0,
+            addons=[BearerAuthAddon(lambda: "token")],
+        )
+
+        response = await executor.execute(request_simple)
+
+        assert response.status == 401
+        assert base.call_count == 2
+
+    async def test_retry_with_bearer_auth_success(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[401, 401, 200])
+        executor = RetryableExecutor(
+            base,
+            max_retries=3,
+            retry_sleep_timeout=0,
+            addons=[BearerAuthAddon(lambda: "token")],
+        )
+
+        response = await executor.execute(request_simple)
+
+        assert response.status == 200
+        assert base.call_count == 3
+
+    async def test_custom_retry_timeout(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[500, 200])
+        executor = RetryableExecutor(
+            base,
+            max_retries=2,
+            retry_sleep_timeout=0,
+            addons=[
+                Retry5xxAddon(retry_timeout=1),
+            ],
+        )
+
+        started_at = time.monotonic()
+        response = await executor.execute(request_simple)
+        elapsed = time.monotonic() - started_at
+
+        assert elapsed >= 1
+
+        assert response.status == 200
+        assert base.call_count == 2
+
+    async def test_timeout_error(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[TimeoutError, TimeoutError(), 200])
+        executor = RetryableExecutor(base, max_retries=3, retry_sleep_timeout=0)
+
+        response = await executor.execute(request_simple)
+
+        assert response.status == 200
+        assert base.call_count == 3
+
+    async def test_timeout_error_propagate(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[TimeoutError, 200])
+        executor = RetryableExecutor(base, max_retries=1, retry_sleep_timeout=0)
+
+        with pytest.raises(ExecuteError) as err:
+            await executor.execute(request_simple)
+
+        assert str(err.value) == "request failed after 1 retries: TimeoutError()"
+
+    async def test_exception_success(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[Exception("some error"), 200])
+        executor = RetryableExecutor(base, max_retries=3, retry_sleep_timeout=0)
+
+        response = await executor.execute(request_simple)
+
+        assert response.status == 200
+        assert base.call_count == 2
+
+    async def test_exception_propagate(self, request_simple: RequestData):
+        base = _DummyExecutor(responses=[Exception("some error"), 200])
+        executor = RetryableExecutor(base, max_retries=1, retry_sleep_timeout=0)
+
+        with pytest.raises(Exception) as err:
+            await executor.execute(request_simple)
+
+        assert str(err.value) == "request failed after 1 retries: Exception(some error)"
+
+    async def test_propagate_execute_error(self, request_simple: RequestData):
+        resp = Response(status=404, backend_response=None, url=request_simple.url)
+        base = _DummyExecutor(responses=[HttpExecuteError(resp), 200])
+        executor = RetryableExecutor(base, max_retries=1, retry_sleep_timeout=0)
+
+        with pytest.raises(ExecuteError) as err:
+            await executor.execute(request_simple)
+
+        assert str(err.value) == "HTTPExecuteError(url=https://example.com, status=404)"
+
+    async def test_with_process_error_addon(self, request_simple: RequestData):
+        class _Addon(Addon):
+            async def process_error(
+                self, request: RequestData, error: Exception
+            ) -> None:
+                raise Exception("error processing exception")
+
+        base = _DummyExecutor(responses=[Exception("some error"), 200])
+        executor = RetryableExecutor(
+            base, max_retries=1, retry_sleep_timeout=0, addons=[_Addon()]
+        )
+
+        with pytest.raises(ExecuteError) as err:
+            await executor.execute(request_simple)
+
+        assert str(err.value) == "request failed after 1 retries: Exception(some error)"
+
+    async def test_with_process_error_addon_success(self, request_simple: RequestData):
+        class _Addon(Addon):
+            async def process_error(
+                self, request: RequestData, error: Exception
+            ) -> None:
+                raise Exception("error processing exception")
+
+        base = _DummyExecutor(responses=[Exception("some error"), 200])
+        executor = RetryableExecutor(
+            base, max_retries=2, retry_sleep_timeout=0, addons=[_Addon()]
+        )
+
+        response = await executor.execute(request_simple)
+
+        assert response.status == 200
+        assert base.call_count == 2