Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrapeURL/pdf): retry

Gergő Móricz7081beff1f96
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 8b42ee716..d0591b57c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -62,6 +62,8 @@ async function scrapePDFWithLlamaParse(meta: Meta, tempFilePath: string): Promis
schema: z.object({
markdown: z.string(),
}),
+ tryCount: 16,
+ tryCooldown: 250,
});
return {
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index 738e240e2..09a280b85 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -15,6 +15,7 @@ export type RobustFetchParams<Schema extends z.Schema<any>> = {
ignoreFailure?: boolean;
requestId?: string;
tryCount?: number;
+ tryCooldown?: number;
};
export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer<Schema>>({
@@ -28,8 +29,9 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer
ignoreFailure = false,
requestId = uuid(),
tryCount = 1,
+ tryCooldown,
}: RobustFetchParams<Schema>): Promise<Output> {
- const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure, tryCount };
+ const params = { url, logger, method, body, headers, schema, ignoreResponse, ignoreFailure, tryCount, tryCooldown };
let request: Response;
try {
@@ -86,6 +88,9 @@ export async function robustFetch<Schema extends z.Schema<any>, Output = z.infer
if (request.status >= 300) {
if (tryCount > 1) {
logger.debug("Request sent failure status, trying " + (tryCount - 1) + " more times", { params, request, response, requestId });
+ if (tryCooldown !== undefined) {
+ await new Promise((resolve) => setTimeout(() => resolve(null), tryCooldown));
+ }
return await robustFetch({
...params,
requestId,