fix: Crawl new chromewebstore.google.com page

Author: aklinker1

src/crawlers/__tests__/__snapshots__/chrome-crawler.test.ts.snap

@@ -0,0 +1,26 @@
+// Bun Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Chrome Web Store Crawler should load and crawl an extension ID correctly 1`] = `
+{
+  "iconUrl": "https://lh3.googleusercontent.com/GcffNyCJaxT2G9dsQCJHhUEMlu_E0vEzph5cLPrQj7UHKat7QyCzGu69Dmp_DDUL8rY-bPMFJceQarS1wcqdwTalTg=s256",
+  "id": "ocfdgncpifmegplaglcnglhioflaimkd",
+  "lastUpdated": "February 4, 2024",
+  "longDescription": 
+"Isn't it annoying when you open a small PR, but when you look at the diff, it's +2000 -16 because you installed a new library? Or what if you had to review that PR, don't line counts like that dissuade you from starting the review?
+In reality, lots of code is generated nowadays and GitHub's line counts are not representative of a PR's true size.
+This extension subtracts generated files from the total line counts, giving you a better idea of how big a PR really is. That's it. That's all it does.
+Generated files are detected from the branch's root .gitattributes file. See GitHub's docs to learn how to mark a file as generated: https://docs.github.com/en/repositories/working-with-files/managing-files/customizing-how-changed-files-appear-on-github
+For a simple example, checkout this extension's .gitattributes file! https://github.com/aklinker1/github-better-line-counts/blob/main/.gitattributes
+---
+The extension is open source. Feel free to contribute if you have any ideas or just star it 😀
+https://github.com/aklinker1/github-better-line-counts"
+,
+  "name": "GitHub: Better Line Counts",
+  "rating": 5,
+  "reviewCount": 2,
+  "shortDescription": "Remove generated files from GitHub line counts",
+  "storeUrl": "https://chromewebstore.google.com/detail/github-better-line-counts/ocfdgncpifmegplaglcnglhioflaimkd",
+  "version": "1.7.1",
+  "weeklyActiveUsers": 73,
+}
+`;

src/crawlers/__tests__/chrome-crawler.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, it } from "bun:test";
+import { crawlExtension } from "../chrome-crawler";
+
+const githubBetterLineCountsId = "ocfdgncpifmegplaglcnglhioflaimkd";
+
+describe("Chrome Web Store Crawler", () => {
+  it("should load and crawl an extension ID correctly", async () => {
+    const res = await crawlExtension(githubBetterLineCountsId, "en");
+
+    expect(res).toMatchSnapshot();
+  });
+});

src/crawlers/chrome-crawler.ts

@@ -1,42 +1,100 @@
 import consola from "consola";
-import { parseHTML } from "linkedom";
+import { HTMLAnchorElement, HTMLElement, parseHTML } from "linkedom";
 
 export async function crawlExtension(
   id: string,
   lang: string
 ): Promise<Gql.ChromeExtension | undefined> {
   consola.info("Crawling " + id);
   const url = `https://chromewebstore.google.com/detail/${id}?hl=${lang}`;
-  const res = await fetch(url);
+  const res = await fetch(url, {
+    headers: {
+      // Without a user agent, the request is stuck in a 302 redirect loop
+      "User-Agent":
+        // Firefox:
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0",
+    },
+  });
   if (res.status !== 200) return;
 
   const html = await res.text();
   const { document } = parseHTML(html);
 
   // Uncomment to debug HTML
-  // consola.info(document.documentElement.outerHTML);
+  // Bun.write("chrome.html", document.documentElement.outerHTML);
 
-  const name = metaContent(document, "itemprop=name");
-  const storeUrl = metaContent(document, "itemprop=url");
-  const iconUrl = metaContent(document, "itemprop=image")?.replace(
+  // Basic metadata
+  const name = metaContent(document, "property=og:title");
+  const storeUrl = metaContent(document, "property=og:url");
+  const iconUrl = metaContent(document, "property=og:image")?.replace(
     /=.+?$/,
     "=s256"
   );
-  const weeklyActiveUsers = metaContent(document, "itemprop=interactionCount")
-    // "UserDownloads:XYZ+"
-    ?.replace("UserDownloads:", "")
-    .replace(",", "")
-    .replace("+", "");
-  const lastUpdated = nextSpanText(document, "Updated:");
-  const version = metaContent(document, "itemprop=version");
   const shortDescription = metaContent(document, "property=og:description");
-  const longDescription = document
-    .querySelector("div[itemprop=description]")
-    ?.nextElementSibling?.textContent?.trim();
 
-  const ratingDiv = document.querySelector(".rsw-stars");
-  const rating = extractNumber(ratingDiv.title); // "Average rating: 4.78 stars"
-  const reviewCount = extractNumber(ratingDiv.textContent); // "(1024)"
+  // Grab the main sections that contain content
+  const sections = (document as HTMLElement).querySelectorAll(
+    "main > * > section"
+  );
+  const header: HTMLElement = sections[0];
+  const description: HTMLElement = sections[2];
+  const details: HTMLElement = sections[3];
+
+  // Header
+
+  // userRowCount.outerHTHML:
+  // <div>
+  //   <a>...</a>
+  //   <a>...</a>
+  //   73 users
+  // </div>
+  // Remove the anchors and extract "73" from the text content
+  const userCountRow = header.querySelector("div:first-child > div:last-child");
+  userCountRow
+    .querySelectorAll("a")
+    .forEach((anchor: HTMLAnchorElement) => anchor.remove());
+  const weeklyActiveUsers = (userCountRow.textContent as string)
+    // "XYZ+ users"
+    .replace(" users", "")
+    .replace(",", "")
+    .replace("+", "")
+    .trim();
+
+  // ratingRow.outerHTML:
+  // <span>
+  //   <span>
+  //     <span>5.0</span>
+  //     <svg ><path /></svg>
+  //     <span>(<a><p>2 ratings</p></a>)</span>
+  //   </span>
+  // </span>
+  const ratingRow = header.querySelector(
+    "div:first-child > div:nth-child(2) > span:last-child"
+  );
+  const rating = extractNumber(
+    ratingRow.querySelector("span:first-child > span:first-child").textContent
+  );
+  const reviewCount = extractNumber(ratingRow.querySelector("p").textContent);
+
+  // Details
+
+  const detailItems = details.querySelectorAll("li > div:last-child");
+  const version = detailItems[0].textContent.trim();
+  const lastUpdated = detailItems[1].textContent.trim();
+
+  // Description
+
+  const longDescription = description
+    .querySelector("p:last-child")
+    .textContent.replaceAll("\n\n", "\n");
+
+  // const longDescription = document
+  //   .querySelector("div[itemprop=description]")
+  //   ?.nextElementSibling?.textContent?.trim();
+  //
+  // const ratingDiv = document.querySelector(".rsw-stars");
+  // const rating = extractNumber(ratingDiv.title); // "Average rating: 4.78 stars"
+  // const reviewCount = extractNumber(ratingDiv.textContent); // "(1024)"
 
   if (name == null) return;
   if (storeUrl == null) return;

src/services/chrome-service.ts

@@ -1,5 +1,5 @@
 import { chrome } from "../crawlers";
-import { createCachedDataLoader, createInMemoryCache } from "../utils/cache";
+import { createCachedDataLoader } from "../utils/cache";
 import { DAY_MS } from "../utils/time";
 
 export function createChromeService() {