Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

proxy used improvement (#1727)

Gergő Móricz916295274431
diff --git a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
index 416e77fc4..11c7304a2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
@@ -14,5 +14,7 @@ export async function scrapeDOCX(meta: Meta, timeToRun: number | undefined): Pro
statusCode: response.status,
html: (await mammoth.convertToHtml({ path: tempFilePath })).value,
+
+ proxyUsed: "basic",
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
index c4e487b84..1e4897cea 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
@@ -120,5 +120,7 @@ export async function scrapeURLWithFetch(
contentType: (response.headers.find(
(x) => x[0].toLowerCase() === "content-type",
) ?? [])[1] ?? undefined,
+
+ proxyUsed: "basic",
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 4a26d8768..68d107fc4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -94,6 +94,8 @@ const successSchema = z.object({
.or(z.null()),
docUrl: z.string().optional(),
+
+ usedMobileProxy: z.boolean().optional(),
});
export type FireEngineCheckStatusSuccess = z.infer<typeof successSchema>;
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 27498958b..a3c0c1561 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -291,6 +291,8 @@ export async function scrapeURLWithFireEngineChromeCDP(
},
}
: {}),
+
+ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
};
}
@@ -354,6 +356,8 @@ export async function scrapeURLWithFireEnginePlaywright(
screenshot: response.screenshots[0],
}
: {}),
+
+ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
};
}
@@ -409,5 +413,7 @@ export async function scrapeURLWithFireEngineTLSClient(
contentType: (Object.entries(response.responseHeaders ?? {}).find(
(x) => x[0].toLowerCase() === "content-type",
) ?? [])[1] ?? undefined,
+
+ proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 5b254259d..5879b70b7 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -119,6 +119,8 @@ export type EngineScrapeResult = {
};
contentType?: string;
+
+ proxyUsed: "basic" | "stealth";
};
const engineHandlers: {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index fb0c765de..1f423f424 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -198,6 +198,8 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult
cacheInfo: {
created_at: new Date(data[0].created_at),
- }
+ },
+
+ proxyUsed: doc.proxyUsed ?? "basic",
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 8bc968130..1135264fa 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -183,6 +183,8 @@ export async function scrapePDF(
html: content,
markdown: content,
+
+ proxyUsed: meta.pdfPrefetch.proxyUsed,
};
} else {
const file = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {
@@ -202,6 +204,8 @@ export async function scrapePDF(
html: content,
markdown: content,
+
+ proxyUsed: "basic",
};
}
}
@@ -300,5 +304,7 @@ export async function scrapePDF(
html: result?.html ?? "",
markdown: result?.markdown ?? "",
numPages: pageCount,
+
+ proxyUsed: "basic",
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
index 13f47fdfd..c38bc0457 100644
--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
@@ -53,5 +53,7 @@ export async function scrapeURLWithPlaywright(
statusCode: response.pageStatusCode,
error: response.pageError,
contentType: response.contentType,
+
+ proxyUsed: "basic",
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index f762cfb1d..02d04aca3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -18,6 +18,7 @@ async function feResToPdfPrefetch(feRes: FireEngineCheckStatusSuccess | undefine
status: feRes.pageStatusCode,
url: feRes.url,
filePath,
+ proxyUsed: feRes.usedMobileProxy ? "stealth" : "basic",
};
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 5e0e97805..31f65617b 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -60,6 +60,7 @@ export type Meta = {
filePath: string;
url?: string;
status: number;
+ proxyUsed: "basic" | "stealth";
} | null | undefined; // undefined: no prefetch yet, null: prefetch came back empty
costTracking: CostTracking;
winnerEngine?: Engine;