Merge pull request #3 from serpapi/support-pagination

Support pagination for `getJson`

Author: sebastianquek

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to
 
 ### Added
 
+- Add pagination support for `getJson`.
 - Export error classes.
 - Add examples.
 - [Apple Reviews] Add more sort options.

README.md

@@ -49,7 +49,7 @@ import { getJson } from "https://deno.land/x/serpapi/mod.ts";
 - Promises and async/await support.
 - Callbacks support.
 - [Examples in JavaScript/TypeScript on Node.js/Deno using ESM/CommonJS, and more](https://github.com/serpapi/serpapi-javascript/tree/master/examples).
-- (Planned) Pagination support.
+- [Pagination support](#pagination).
 - (Planned) More error classes.
 
 ## Configuration
@@ -72,6 +72,33 @@ await getJson("google", { q: "coffee" }); // uses the API key defined in the con
 await getJson("google", { api_key: API_KEY_2, q: "coffee" }); // API_KEY_2 will be used
 ```
 
+## Pagination
+
+Search engines handle pagination in several different ways. Some rely on an
+"offset" value to return results starting from a specific index, while some
+others rely on the typical notion of a "page". These are often combined with a
+"size" value to define how many results are returned in a search.
+
+This module helps you handle pagination easily. After receiving search results
+from `getJson`, simply call the `next()` method on the returned object to
+retrieve the next page of results. If there is no `next()` method, then either
+pagination is not supported for the search engine or there are no more pages to
+be retrieved.
+
+```js
+const page1 = await getJson("google", { q: "coffee", start: 15 });
+const page2 = await page1.next?.();
+```
+
+You may pass in the engine's supported pagination parameters as per normal. In
+the above example, the first page contains the 15th to the 24th result while the
+second page contains the 25th to the 34th result.
+
+Note that if you set `no_cache` to `true`, all subsequent `next()` calls will
+not return cached results.
+
+Refer to the [`getJson` definition below](#getjson) for more examples.
+
 ## Functions
 
 <!-- Generated by documentation.js. Update this documentation by updating the source code. -->
@@ -102,6 +129,8 @@ await getJson("google", { api_key: API_KEY_2, q: "coffee" }); // API_KEY_2 will
 Get a JSON response based on search parameters.
 
 - Accepts an optional callback.
+- Get the next page of results by calling the `.next()` method on the returned
+  response object.
 
 #### Parameters
 
@@ -116,13 +145,50 @@ Get a JSON response based on search parameters.
 #### Examples
 
 ```javascript
-// async/await
+// single call (async/await)
 const json = await getJson("google", { api_key: API_KEY, q: "coffee" });
 
-// callback
+// single call (callback)
 getJson("google", { api_key: API_KEY, q: "coffee" }, console.log);
 ```
 
+```javascript
+// pagination (async/await)
+const page1 = await getJson("google", { q: "coffee", start: 15 });
+const page2 = await page1.next?.();
+```
+
+```javascript
+// pagination (callback)
+getJson("google", { q: "coffee", start: 15 }, (page1) => {
+  page1.next?.((page2) => {
+    console.log(page2);
+  });
+});
+```
+
+```javascript
+// pagination loop (async/await)
+const organicResults = [];
+let page = await getJson("google", { api_key: API_KEY, q: "coffee" });
+while (page) {
+  organicResults.push(...page.organic_results);
+  if (organicResults.length >= 30) break;
+  page = await page.next?.();
+}
+```
+
+```javascript
+// pagination loop (callback)
+const organicResults = [];
+getJson("google", { api_key: API_KEY, q: "coffee" }, (page) => {
+  organicResults.push(...page.organic_results);
+  if (organicResults.length < 30 && page.next) {
+    page.next();
+  }
+});
+```
+
 ### getHtml
 
 Get a HTML response based on search parameters.

src/serpapi.ts

@@ -7,7 +7,12 @@ import {
   LocationsApiParameters,
 } from "./types.ts";
 import { EngineMap } from "./engines/engine_map.ts";
-import { _internals, execute } from "./utils.ts";
+import {
+  _internals,
+  execute,
+  extractNextParameters,
+  haveParametersChanged,
+} from "./utils.ts";
 import { validateApiKey, validateTimeout } from "./validators.ts";
 
 const ACCOUNT_PATH = "/account";
@@ -18,24 +23,57 @@ const SEARCH_ARCHIVE_PATH = `/searches`;
 /**
  * Get a JSON response based on search parameters.
  * - Accepts an optional callback.
+ * - Get the next page of results by calling the `.next()` method on the returned response object.
  *
  * @param {string} engine - engine name
  * @param {object} parameters - search query parameters for the engine
  * @param {fn=} callback - optional callback
  * @example
- * // async/await
+ * // single call (async/await)
  * const json = await getJson("google", { api_key: API_KEY, q: "coffee" });
  *
- * // callback
+ * // single call (callback)
  * getJson("google", { api_key: API_KEY, q: "coffee" }, console.log);
+ *
+ * @example
+ * // pagination (async/await)
+ * const page1 = await getJson("google", { q: "coffee", start: 15 });
+ * const page2 = await page1.next?.();
+ *
+ * @example
+ * // pagination (callback)
+ * getJson("google", { q: "coffee", start: 15 }, (page1) => {
+ *   page1.next?.((page2) => {
+ *     console.log(page2);
+ *   });
+ * });
+ *
+ * @example
+ * // pagination loop (async/await)
+ * const organicResults = [];
+ * let page = await getJson("google", { api_key: API_KEY, q: "coffee" });
+ * while (page) {
+ *   organicResults.push(...page.organic_results);
+ *   if (organicResults.length >= 30) break;
+ *   page = await page.next?.();
+ * }
+ *
+ * @example
+ * // pagination loop (callback)
+ * const organicResults = [];
+ * getJson("google", { api_key: API_KEY, q: "coffee" }, (page) => {
+ *   organicResults.push(...page.organic_results);
+ *   if (organicResults.length < 30 && page.next) {
+ *     page.next();
+ *   }
+ * });
  */
 export async function getJson<
   E extends keyof EngineMap,
-  R extends BaseResponse<EngineMap[E]["parameters"]>,
 >(
   engine: E,
   parameters: EngineMap[E]["parameters"],
-  callback?: (json: R) => void,
+  callback?: (json: BaseResponse<EngineMap[E]["parameters"]>) => void,
 ) {
   const key = validateApiKey(parameters.api_key, true);
   const timeout = validateTimeout(parameters.timeout);
@@ -49,7 +87,26 @@ export async function getJson<
     },
     timeout,
   );
-  const json = await response.json() as R;
+  const json = await response.json() as BaseResponse<
+    EngineMap[E]["parameters"]
+  >;
+  const nextParametersFromResponse = extractNextParameters<E>(json);
+  if (
+    // https://github.com/serpapi/public-roadmap/issues/562
+    // https://github.com/serpapi/public-roadmap/issues/563
+    engine !== "yahoo_shopping" &&
+    nextParametersFromResponse
+  ) {
+    const nextParameters = { ...parameters, ...nextParametersFromResponse };
+    if (haveParametersChanged(parameters, nextParameters)) {
+      json.next = (innerCallback = callback) =>
+        getJson(
+          engine,
+          nextParameters,
+          innerCallback,
+        );
+    }
+  }
   callback?.(json);
   return json;
 }

src/types.ts

@@ -52,6 +52,11 @@ export type BaseResponse<P = Record<string | number | symbol, never>> = {
   search_parameters:
     & { engine: string }
     & Omit<BaseParameters & P, "api_key" | "no_cache" | "async" | "timeout">;
+  serpapi_pagination?: { next: string };
+  pagination?: { next: string };
+  next?: (
+    callback?: (json: BaseResponse<P>) => void,
+  ) => Promise<BaseResponse<P>>;
   // deno-lint-ignore no-explicit-any
   [key: string]: any; // TODO(seb): use recursive type
 };

src/utils.ts

@@ -1,3 +1,4 @@
+import type { EngineMap } from "./engines/engine_map.ts";
 import { version } from "../version.ts";
 
 type UrlParameters = Record<
@@ -34,6 +35,44 @@ function getBaseUrl() {
   return "https://serpapi.com";
 }
 
+type NextParametersKeys<E extends keyof EngineMap> = Omit<
+  EngineMap[E]["parameters"],
+  "api_key" | "no_cache" | "async" | "timeout"
+>;
+type NextParameters<E extends keyof EngineMap> = {
+  [K in keyof NextParametersKeys<E>]: string;
+};
+export function extractNextParameters<
+  E extends keyof EngineMap,
+>(json: {
+  serpapi_pagination?: { next: string };
+  pagination?: { next: string };
+}) {
+  const nextUrlString = json["serpapi_pagination"]?.["next"] ||
+    json["pagination"]?.["next"];
+
+  if (nextUrlString) {
+    const nextUrl = new URL(nextUrlString);
+    const nextParameters = Object.fromEntries(nextUrl.searchParams.entries());
+    delete nextParameters["engine"];
+    return nextParameters as NextParameters<E>;
+  }
+}
+
+export function haveParametersChanged(
+  parameters: Record<string, unknown>,
+  nextParameters: Record<string, unknown>,
+) {
+  const keys = [
+    ...Object.keys(parameters),
+    ...Object.keys(nextParameters),
+  ];
+  const uniqueKeys = new Set(keys);
+  return [...uniqueKeys].some((key) =>
+    `${parameters[key]}` !== `${nextParameters[key]}` // string comparison
+  );
+}
+
 function getSource() {
   const moduleSource = `serpapi@${version}`;
   try {