Retry without cache if received anything in 403

Author: PrashamTrivedi

deno.lock

@@ -12,22 +12,39 @@ async function jinaFetch(endpoint: string, url: string): Promise<string> {
 
     const jinaApiEndpoint = `https://${endpoint}.jina.ai/${url}`
     log.info(`Fetching ${jinaApiEndpoint}`)
-    const response = await fetch(jinaApiEndpoint, {
-        headers: {
-            Authorization: `Bearer ${getJinaApiKey()}`
-        }
-    })
+
+    const headers = {
+        Authorization: `Bearer ${getJinaApiKey()}`,
+        'X-Return-Format': 'markdown',
+
+    }
+
+    let response = await fetch(jinaApiEndpoint, {headers})
+    log.info({headers, success: response.ok, code: response.status})
+
+    // If we get a 403, retry once with X-No-Cache header
+    if (response.status === 403) {
+        log.info('Got 403, retrying with X-No-Cache header')
+        response = await fetch(jinaApiEndpoint, {
+            headers: {
+                ...headers,
+                'X-No-Cache': 'true'
+            }
+        })
+        console.log({headers})
+    }
 
     if (!response.ok) {
         throw new Error(`Jina API error: ${response.statusText}`)
     }
     log.info(`fetched ${jinaApiEndpoint}`)
+    log.info({headers, success: response.ok, code: response.status})
 
     return await response.text()
 }
 
 export async function readPage(url: string): Promise<string> {
-    return await jinaFetch('r', encodeURIComponent(url))
+    return await jinaFetch('r', url)
 }
 
 export async function search(searchTerm: string): Promise<string> {