Refactor.

Author: toddbirchard

README.md

@@ -11,7 +11,7 @@
 
 Make asynchronous HTTP requests and write to disk using **asyncio**, **aiohttp**, & **aiofiles**.
 
-* **Tutorial**: https://hackersandslackers.com/concurrency-asyncio-aiohttp-aiofiles/
+* **Tutorial**: https://hackersandslackers.com/async-requests-with-aiohttp/
 * **Demo**: TBD
 
 

asyncio_tutorial/__init__.py

@@ -1,6 +1,6 @@
 """Script initialization."""
 import asyncio
-from asyncio import Task
+from asyncio import SelectorEventLoop, Task
 from typing import List
 
 from aiohttp import ClientSession
@@ -19,12 +19,20 @@ async def init_script():
     async with ClientSession(headers=HTML_HEADERS) as session:
         tasks = await create_tasks(session, urls, HTML_EXPORT_DIR)
         await asyncio.gather(*tasks)
+        loop = inspect_loop(tasks)
         LOGGER.success(
-            f"Successfully saved {len(urls)} HTML pages to `{HTML_EXPORT_DIR}`"
+            f"Saved {len(urls)} HTML pages to `{HTML_EXPORT_DIR}`. Loop: {loop}"
         )
 
 
-def inspect_loop(tasks: List[Task]):
-    """Get async loop info."""
+def inspect_loop(tasks: List[Task]) -> SelectorEventLoop:
+    """
+    Get async loop info.
+
+    :param: List[Task] tasks
+
+    :return: SelectorEventLoop
+    """
     loop = tasks[0].get_loop()
     LOGGER.info(f"Loop info: {loop}")
+    return loop

asyncio_tutorial/fetcher.py

@@ -2,7 +2,7 @@
 from aiohttp import ClientError, ClientSession, InvalidURL
 
 from asyncio_tutorial.logger import LOGGER
-from asyncio_tutorial.writer import write_html_file
+from asyncio_tutorial.writer import write_fetched_file_locally
 
 
 async def fetch_and_save_url(
@@ -20,9 +20,14 @@ async def fetch_and_save_url(
     """
     try:
         async with session.get(url) as resp:
-            text = await resp.read()
+            body = await resp.read()
+            filetype = (
+                resp.headers.get("Content-Type")
+                .replace("; charset=UTF-8", "")
+                .replace("text/", "")
+            )
             LOGGER.info(f"Successfully fetched URL {i + 1} of {total_count}: {url}")
-            await write_html_file(url, text, directory)
+            await write_fetched_file_locally(url, body, filetype, directory)
     except InvalidURL as e:
         LOGGER.error(f"Unable to fetch invalid URL `{url}`: {e}")
     except ClientError as e:

asyncio_tutorial/writer.py

@@ -4,18 +4,21 @@
 from .logger import LOGGER
 
 
-async def write_html_file(url: str, text: bytes, directory: str):
+async def write_fetched_file_locally(
+    url: str, body: bytes, filetype: str, directory: str
+):
     """
     Write contents of fetched URL to new file in local directory.
 
     :param str url: URL which was fetched.
-    :param bytes text: Source HTML of a single fetched URL.
+    :param bytes body: Source HTML of a single fetched URL.
+    :param str filetype: File extension to save fetched data as.
     :param str directory: Local directory to save exports to.
     """
     try:
-        filename = f"{directory}/{url.split('/')[-2]}.html"
+        filename = f"{directory}/{url.split('/')[-2]}.{filetype}"
         async with aiofiles.open(filename, mode="wb+") as f:
-            await f.write(text)
+            await f.write(body)
             await f.close()
     except Exception as e:
-        LOGGER.error(f"Unexpected error while writing file from `{url}`: {e}")
+        LOGGER.error(f"Unexpected error while writing from `{url}`: {e}")

poetry.lock

@@ -30,9 +30,9 @@ loguru = "*"
 [tool.poetry.dev-dependencies]
 pytest = "*"
 mypy = "*"
-black = "^21.7b0"
+black = "^21.9b0"
 isort = "5.9.3"
-flake8 = "^3.9.0"
+flake8 = "^3.9.2"
 
 [tool.poetry.scripts]
 run = "main:init_script"