Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

fix(crawler): proper handling for proxy auth with CDP interceptions

Mohamed Bassemb9b252ecb6d2
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index e71eaff4..87768507 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -596,6 +596,7 @@ async function crawlPage(
userAgent:
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
proxy: proxyConfig,
+ serviceWorkers: "block",
}),
);
@@ -641,6 +642,25 @@ async function crawlPage(
// Ignore errors — the request may have been canceled.
});
};
+ cdpSession.on("Fetch.authRequired", async (event) => {
+ const authChallengeResponse =
+ event.authChallenge.source === "Proxy" &&
+ (proxyConfig?.username || proxyConfig?.password)
+ ? {
+ response: "ProvideCredentials" as const,
+ username: proxyConfig.username ?? "",
+ password: proxyConfig.password ?? "",
+ }
+ : { response: "Default" as const };
+ await cdpSession
+ ?.send("Fetch.continueWithAuth", {
+ requestId: event.requestId,
+ authChallengeResponse,
+ })
+ .catch(() => {
+ // Ignore errors — the request may have been canceled.
+ });
+ });
cdpSession.on("Fetch.requestPaused", async (event) => {
try {
const status = event.responseStatusCode;
@@ -690,7 +710,11 @@ async function crawlPage(
}
});
await cdpSession.send("Fetch.enable", {
- patterns: [{ urlPattern: "*", requestStage: "Response" }],
+ handleAuthRequests: true,
+ patterns: [
+ { urlPattern: "*", requestStage: "Request" },
+ { urlPattern: "*", requestStage: "Response" },
+ ],
});
} catch (e) {
logger.warn(