Refactor.

Author: toddbirchard

.github/workflows/isort.yml

@@ -0,0 +1,15 @@
+name: Run isort
+on:
+  - push
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - uses: jamescurtin/isort-action@master
+        with:
+            requirementsFiles: "requirements.txt"

asyncio_tutorial/fetcher.py

@@ -1,8 +1,8 @@
-"""Fetch and save HTML pages asynchronously."""
+"""Fetch and save files to disk asynchronously."""
 from aiohttp import ClientError, ClientSession, InvalidURL
 
 from asyncio_tutorial.logger import LOGGER
-from asyncio_tutorial.writer import write_fetched_file_locally
+from asyncio_tutorial.writer import write_file
 
 
 async def fetch_and_save_url(
@@ -27,7 +27,7 @@ async def fetch_and_save_url(
                 .replace("text/", "")
             )
             LOGGER.info(f"Successfully fetched URL {i + 1} of {total_count}: {url}")
-            await write_fetched_file_locally(url, body, filetype, directory)
+            await write_file(url, body, filetype, directory)
     except InvalidURL as e:
         LOGGER.error(f"Unable to fetch invalid URL `{url}`: {e}")
     except ClientError as e:

asyncio_tutorial/logger.py

@@ -1,8 +1,11 @@
-"""Create custom logger."""
+"""Custom logger."""
+from os import path
 from sys import stdout
 
 from loguru import logger as custom_logger
 
+from config import ENVIRONMENT
+
 
 def formatter(log: dict) -> str:
     """
@@ -46,6 +49,14 @@ def create_logger() -> custom_logger:
     """
     custom_logger.remove()
     custom_logger.add(stdout, colorize=True, format=formatter)
+    if ENVIRONMENT == "production":
+        # Datadog JSON logs
+        custom_logger.add(
+            "/var/log/asyncio_tutorial/info.json",
+            format=formatter,
+            rotation="500 MB",
+            compression="zip",
+        )
     return custom_logger
 
 

asyncio_tutorial/tasks.py

@@ -1,4 +1,4 @@
-"""Prepare tasks to be executed asynchronously."""
+"""Prepare tasks to be executed."""
 import asyncio
 from asyncio import Task
 from typing import List

asyncio_tutorial/writer.py

@@ -4,9 +4,7 @@
 from .logger import LOGGER
 
 
-async def write_fetched_file_locally(
-    url: str, body: bytes, filetype: str, directory: str
-):
+async def write_file(url: str, body: bytes, filetype: str, directory: str):
     """
     Write contents of fetched URL to new file in local directory.
 

config.py

@@ -16,3 +16,8 @@
     "connection": "keep-alive",
     "accept": "*/*",
 }
+
+# Determine environment based on log folder
+ENVIRONMENT = "development"
+if path.isdir("/var/log/asyncio_tutorial"):
+    ENVIRONMENT = "production"

poetry.lock

@@ -43,3 +43,7 @@ issues = "https://github.com/hackersandslackers/asyncio-tutorial/issues"
 [build-system]
 requires = ["poetry>=1.18"]
 build-backend = "poetry.masonry.api"
+
+[tool.isort]
+profile = "black"
+src_paths = ["asyncio_tutorial", "data", "main", "config"]