Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/pdf): retry
Gergő Móricz7081beff1f96
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 8b42ee716..d0591b57c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -62,6 +62,8 @@ async function scrapePDFWithLlamaParse(meta: Meta, tempFilePath: string): Promisschema: z.object({markdown: z.string(),}),+ tryCount: 16,+ tryCooldown: 250,});return {diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex 738e240e2..09a280b85 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -15,6 +15,7 @@ export type RobustFetchParams<Schema extends z.Schema<any>> = {ignoreFailure?: boolean;requestId?: string;tryCount?: number;+ tryCooldown?: number;};export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer<Schema>>({@@ -28,8 +29,9 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.inferignoreFailure = false,requestId = uuid(),tryCount = 1,+ tryCooldown,}: RobustFetchParams<Schema>): Promise<Output> {- const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure, tryCount };+ const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure, tryCount, tryCooldown };let request: Response;try {@@ -86,6 +88,9 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.inferif (request.status >= 300) {if (tryCount > 1) {logger.debug("Request sent failure status, trying " + (tryCount - 1) + " more times", { params, request, response, requestId });+ if (tryCooldown !== undefined) {+ await new Promise((resolve) => setTimeout(() => resolve(null), tryCooldown));+ }return await robustFetch({...params,requestId,