Add pytest config; tweak naming schemes.

Author: toddbirchard

aiohttp_aiofiles_tutorial/__init__.py

@@ -9,7 +9,7 @@
 from config import EXPORT_FILE, HTML_HEADERS
 from logger import LOGGER
 
-from .data import urls
+from .data import urls_to_fetch
 from .tasks import create_tasks
 
 
@@ -30,5 +30,5 @@ async def execute_fetcher_tasks(outfile: AsyncIOFile):
     :param AsyncIOFile outfile: Path of local file to write to.
     """
     async with ClientSession(headers=HTML_HEADERS) as session:
-        task_list = await create_tasks(session, urls, outfile)
+        task_list = await create_tasks(session, urls_to_fetch, outfile)
         await asyncio.gather(*task_list)

aiohttp_aiofiles_tutorial/data/__init__.py

@@ -1,6 +1,6 @@
 """Parse data from local files."""
 from config import CSV_FILEPATH
 
-from .parser import parse_urls_from_csv
+from .urls import parse_urls_from_csv
 
-urls = parse_urls_from_csv(CSV_FILEPATH)
+urls_to_fetch = parse_urls_from_csv(CSV_FILEPATH)

aiohttp_aiofiles_tutorial/data/tests/test_parser.py

@@ -1,12 +1,13 @@
 """Test reading tutorial sample data URLs."""
-from aiohttp_aiofiles_tutorial.data.parser import parse_urls_from_csv
 from config import CSV_FILEPATH
 
+from aiohttp_aiofiles_tutorial.data.urls import parse_urls_from_csv
+
 
 def test_parse_urls():
     """Ensure URL sample data is parsed correctly."""
 
-    urls = parse_urls_from_csv(CSV_FILEPATH)
+    urls_to_fetch = parse_urls_from_csv(CSV_FILEPATH)
 
-    assert (type(urls)) == list
-    assert "https://" in urls[0]
+    assert (type(urls_to_fetch)) == list
+    assert "https://" in urls_to_fetch[0]

aiohttp_aiofiles_tutorial/data/urls.csv

@@ -13,8 +13,8 @@ def parse_urls_from_csv(filepath: str) -> List[str]:
     """
     urls = []
     with open(filepath, newline="") as f:
-        reader = csv.reader(f)
+        reader = csv.DictReader(f)
         for line in reader:
-            urls.append(line[0])
+            urls.append(line["url"])
 
     return urls

aiohttp_aiofiles_tutorial/data/parser.py aiohttp_aiofiles_tutorial/data/urls.pyaiohttp_aiofiles_tutorial/data/parser.py renamed to aiohttp_aiofiles_tutorial/data/urls.py

@@ -23,13 +23,15 @@ async def parse_html_page_data(html: str, url: str) -> str:
             .replace("'", "`")
         )
         primary_tag = soup.head.select_one("meta[property='article:tag']").get("content")
-        published_at = soup.head.select_one("meta[property='article:published_time']").get(
-            "content"
-        ).split("T")[0]
+        published_at = (
+            soup.head.select_one("meta[property='article:published_time']")
+            .get("content")
+            .split("T")[0]
+        )
         if primary_tag is None:
             primary_tag = ""
         return f"{title}, {description}, {primary_tag}, {url}, {published_at}"
     except ValueError as e:
-        LOGGER.error(f"Parsing failed for {url}: {e}")
+        LOGGER.error(f"ValueError occurred when parsing html for {url}: {e}")
     except Exception as e:
-        LOGGER.error(f"Parsing failed for {url}: {e}")
+        LOGGER.error(f"Parsing failed when parsing html for {url}: {e}")

aiohttp_aiofiles_tutorial/parser.py

@@ -1,11 +1,13 @@
 """Test metadata parser accuracy with local HTML file."""
 import aiofiles
 import pytest
-from aiohttp_aiofiles_tutorial.parser import parse_html_page_data
+import pytest_asyncio
 from config import BASE_DIR
 
+from aiohttp_aiofiles_tutorial.parser import parse_html_page_data
+
 
-@pytest.fixture
+@pytest_asyncio.fixture
 async def sample_page_metadata():
     """Expected metadata to be returned from parsing `intro_to_asyncio.html`"""
     title = "Intro to Asynchronous Python with Asyncio"
@@ -25,3 +27,4 @@ async def test_parse_html_page_data(sample_page_metadata):
         url = "https://hackersandslackers.com/intro-to-asyncio-concurrency/"
         metadata = await parse_html_page_data(html, url)
         assert metadata == sample_page_metadata
+        await file.close()

aiohttp_aiofiles_tutorial/tests/test_parser.py

@@ -52,4 +52,4 @@ profile = "black"
 src_paths = ["aiohttp_aiofiles_tutorial"]
 
 [tool.black]
-line-length = 100
+line-length = 100

poetry.lock

@@ -0,0 +1,2 @@
+[pytest]
+asyncio_mode = auto