Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
proxy used improvement (#1727)
Gergő Móricz916295274431
diff --git a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts b/apps/api/src/scraper/scrapeURL/engines/docx/index.tsindex 416e77fc4..11c7304a2 100644--- a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts@@ -14,5 +14,7 @@ export async function scrapeDOCX(meta: Meta, timeToRun: number | undefined): ProstatusCode: response.status,html: (await mammoth.convertToHtml({ path: tempFilePath })).value,++ proxyUsed: "basic",};}diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.tsindex c4e487b84..1e4897cea 100644--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts@@ -120,5 +120,7 @@ export async function scrapeURLWithFetch(contentType: (response.headers.find((x) => x[0].toLowerCase() === "content-type",) ?? [])[1] ?? undefined,++ proxyUsed: "basic",};}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 4a26d8768..68d107fc4 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -94,6 +94,8 @@ const successSchema = z.object({.or(z.null()),docUrl: z.string().optional(),++ usedMobileProxy: z.boolean().optional(),});export type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 27498958b..a3c0c1561 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -291,6 +291,8 @@ export async function scrapeURLWithFireEngineChromeCDP(},}: {}),++ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",};}@@ -354,6 +356,8 @@ export async function scrapeURLWithFireEnginePlaywright(screenshot: response.screenshots[0],}: {}),++ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",};}@@ -409,5 +413,7 @@ export async function scrapeURLWithFireEngineTLSClient(contentType: (Object.entries(response.responseHeaders ?? {}).find((x) => x[0].toLowerCase() === "content-type",) ?? [])[1] ?? undefined,++ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",};}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 5b254259d..5879b70b7 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -119,6 +119,8 @@ export type EngineScrapeResult = {};contentType?: string;++ proxyUsed: "basic" | "stealth";};const engineHandlers: {diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex fb0c765de..1f423f424 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -198,6 +198,8 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResultcacheInfo: {created_at: new Date(data[0].created_at),- }+ },++ proxyUsed: doc.proxyUsed ?? "basic",};}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 8bc968130..1135264fa 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -183,6 +183,8 @@ export async function scrapePDF(html: content,markdown: content,++ proxyUsed: meta.pdfPrefetch.proxyUsed,};} else {const file = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {@@ -202,6 +204,8 @@ export async function scrapePDF(html: content,markdown: content,++ proxyUsed: "basic",};}}@@ -300,5 +304,7 @@ export async function scrapePDF(html: result?.html ?? "",markdown: result?.markdown ?? "",numPages: pageCount,++ proxyUsed: "basic",};}diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.tsindex 13f47fdfd..c38bc0457 100644--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts@@ -53,5 +53,7 @@ export async function scrapeURLWithPlaywright(statusCode: response.pageStatusCode,error: response.pageError,contentType: response.contentType,++ proxyUsed: "basic",};}diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex f762cfb1d..02d04aca3 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -18,6 +18,7 @@ async function feResToPdfPrefetch(feRes: FireEngineCheckStatusSuccess | undefinestatus: feRes.pageStatusCode,url: feRes.url,filePath,+ proxyUsed: feRes.usedMobileProxy ? "stealth" : "basic",};}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 5e0e97805..31f65617b 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -60,6 +60,7 @@ export type Meta = {filePath: string;url?: string;status: number;+ proxyUsed: "basic" | "stealth";} | null | undefined; // undefined: no prefetch yet, null: prefetch came back emptycostTracking: CostTracking;winnerEngine?: Engine;