A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
feat(workers): wait for self-clearing bot challenges before capturing
Mohamed Bassem18003460c6fa
diff --git a/apps/workers/workers/crawler/challenge.ts b/apps/workers/workers/crawler/challenge.tsnew file mode 100644index 00000000..939b9844--- /dev/null+++ b/apps/workers/workers/crawler/challenge.ts@@ -0,0 +1,97 @@+// Waits out non-interactive bot-protection challenges (e.g. Cloudflare's+// "Just a moment..." page or DataDome's device check). These interstitials run+// some JS and then reload into the real page on their own when the browser+// passes. Capturing right after the initial load would store the challenge+// page instead, so when one is detected we give it a bounded amount of time to+// clear.+import type { Page } from "patchright";+import { abortRace, raceWith } from "utils";++import { setSpanAttributes } from "@karakeep/shared-server";+import serverConfig from "@karakeep/shared/config";+import logger from "@karakeep/shared/logger";++import {+ isLikelyChallengePage,+ isWaitableChallenge,+} from "../utils/metadataResolver";++// Statuses bot-protection vendors serve their interstitials with.+const CHALLENGE_STATUS_CODES = new Set([401, 403, 429, 503]);++const POLL_INTERVAL_MS = 500;+const SETTLE_TIMEOUT_MS = 5000;++async function pageIsWaitableChallenge(page: Page): Promise<boolean> {+ try {+ const [title, htmlContent] = await Promise.all([+ page.title(),+ page.content(),+ ]);+ return isWaitableChallenge(title, htmlContent);+ } catch {+ // The page is navigating (typically the challenge reloading into the real+ // page); treat it as still pending and check again on the next poll.+ return true;+ }+}++/**+ * If the page looks like a self-clearing bot challenge, waits (up to+ * CRAWLER_CHALLENGE_WAIT_SEC) for it to go away and for the resulting page to+ * settle. Returns without waiting for regular pages.+ */+export async function waitForChallengeToClear(+ page: Page,+ statusCode: number,+ jobId: string,+ abortSignal: AbortSignal,+): Promise<void> {+ const waitMs = serverConfig.crawler.challengeWaitSec * 1000;+ if (waitMs <= 0) {+ return;+ }+ // Cheap gate so regular pages don't pay for serializing their DOM.+ const title = await page.title().catch(() => "");+ if (+ !CHALLENGE_STATUS_CODES.has(statusCode) &&+ !isLikelyChallengePage({ title })+ ) {+ return;+ }+ if (!(await pageIsWaitableChallenge(page))) {+ return;+ }++ logger.info(+ `[Crawler][${jobId}] The page looks like a bot challenge, waiting up to ${serverConfig.crawler.challengeWaitSec}s for it to clear`,+ );+ const startedAt = Date.now();+ const deadline = startedAt + waitMs;+ let cleared = false;+ while (Date.now() < deadline) {+ await raceWith<unknown>(+ new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)),+ abortRace(abortSignal),+ );+ if (!(await pageIsWaitableChallenge(page))) {+ cleared = true;+ break;+ }+ }++ if (cleared) {+ await page+ .waitForLoadState("networkidle", { timeout: SETTLE_TIMEOUT_MS })+ .catch(() => undefined);+ }+ abortSignal.throwIfAborted();+ const waitedMs = Date.now() - startedAt;+ setSpanAttributes({+ "crawler.challenge.cleared": cleared,+ "crawler.challenge.waitMs": waitedMs,+ });+ logger.info(+ `[Crawler][${jobId}] Bot challenge ${cleared ? "cleared" : "did not clear"} after ${waitedMs}ms`,+ );+}diff --git a/apps/workers/workers/crawler/crawlPage.ts b/apps/workers/workers/crawler/crawlPage.tsindex 93522d25..82692ea6 100644--- a/apps/workers/workers/crawler/crawlPage.ts+++ b/apps/workers/workers/crawler/crawlPage.ts@@ -47,6 +47,7 @@ import {trackContext,untrackContext,} from "./browser";+import { waitForChallengeToClear } from "./challenge";import { truncateUrl } from "./utils";const tracer = getTracer("@karakeep/workers");@@ -695,6 +696,18 @@ export async function crawlPage();}const targetUrl = navigationValidation.url.toString();+ // Tracks the status of the latest main-frame document, which differs+ // from the initial navigation's when a bot challenge clears and+ // reloads into the real page.+ let mainDocumentStatus: number | undefined;+ activePage.on("response", (res) => {+ if (+ res.request().isNavigationRequest() &&+ res.frame() === activePage.mainFrame()+ ) {+ mainDocumentStatus = res.status();+ }+ });logger.info(`[Crawler][${jobId}] Navigating to "${targetUrl}"`);const response = await withSpan(tracer,@@ -751,9 +764,21 @@ export async function crawlPage(setup.autoconsentEnabled,abortSignal,);-abortSignal.throwIfAborted();+ await withSpan(+ tracer,+ "crawlerWorker.crawlPage.waitForChallenge",+ { attributes: { "job.id": jobId } },+ () =>+ waitForChallengeToClear(+ activePage,+ mainDocumentStatus ?? response?.status() ?? 0,+ jobId,+ abortSignal,+ ),+ );+logger.info(`[Crawler][${jobId}] Finished waiting for the page to load.`,);@@ -767,7 +792,7 @@ export async function crawlPage(return {htmlContent,- statusCode: response?.status() ?? 0,+ statusCode: mainDocumentStatus ?? response?.status() ?? 0,screenshot,pdf,url: activePage.url(),diff --git a/apps/workers/workers/utils/metadataResolver.test.ts b/apps/workers/workers/utils/metadataResolver.test.tsindex 6cce8021..0801ccec 100644--- a/apps/workers/workers/utils/metadataResolver.test.ts+++ b/apps/workers/workers/utils/metadataResolver.test.ts@@ -1,6 +1,10 @@import { describe, expect, it } from "vitest";-import { isLikelyChallengePage, resolveMetadata } from "./metadataResolver";+import {+ isLikelyChallengePage,+ isWaitableChallenge,+ resolveMetadata,+} from "./metadataResolver";function meta(fields: Record<string, string | null | undefined>) {return {@@ -139,3 +143,41 @@ describe("isLikelyChallengePage", () => {);});});++describe("isWaitableChallenge", () => {+ it("waits on Cloudflare's interstitial", () => {+ expect(+ isWaitableChallenge(+ "Just a moment...",+ "<html><head><title>Just a moment...</title></head><body><script>window._cf_chl_opt={}</script></body></html>",+ ),+ ).toBe(true);+ });++ it("waits on DataDome's device check", () => {+ expect(+ isWaitableChallenge(+ "wsj.com",+ "<html><body><script>var dd={'rt':'i','cid':'x','host':'geo.captcha-delivery.com'}</script></body></html>",+ ),+ ).toBe(true);+ });++ it("doesn't wait on DataDome's captcha, which needs a human", () => {+ expect(+ isWaitableChallenge(+ "wsj.com",+ "<html><body><script>var dd={'rt':'c','cid':'x','host':'geo.captcha-delivery.com'}</script></body></html>",+ ),+ ).toBe(false);+ });++ it("doesn't wait on regular pages", () => {+ expect(+ isWaitableChallenge(+ "Example Domain",+ "<html><head><title>Example Domain</title></head><body><p>Hello</p></body></html>",+ ),+ ).toBe(false);+ });+});diff --git a/apps/workers/workers/utils/metadataResolver.ts b/apps/workers/workers/utils/metadataResolver.tsindex 35b9b300..37cea2f3 100644--- a/apps/workers/workers/utils/metadataResolver.ts+++ b/apps/workers/workers/utils/metadataResolver.ts@@ -82,6 +82,24 @@ export function isLikelyChallengePage({return CHALLENGE_PAGE_BODY_MARKERS.some((marker) => lowered.includes(marker));}+// DataDome's captcha page (as opposed to its self-clearing device check) needs+// a human to solve it, so there's no point waiting for it to clear.+const DATADOME_HARD_CAPTCHA = /['"]rt['"]\s*:\s*['"]c['"]/;++/**+ * Whether the page is a bot challenge that may clear by itself (and reload+ * into the real page) if the browser keeps waiting.+ */+export function isWaitableChallenge(+ title: string,+ htmlContent: string,+): boolean {+ return (+ isLikelyChallengePage({ title, htmlContent }) &&+ !DATADOME_HARD_CAPTCHA.test(htmlContent)+ );+}+/*** Merges the browser render's metadata with the preflight probe's, per field.* The render normally wins (it sees the post-JS DOM), with the probe fillingdiff --git a/docs/docs/03-configuration/01-environment-variables.md b/docs/docs/03-configuration/01-environment-variables.mdindex d2297b0d..bec66794 100644--- a/docs/docs/03-configuration/01-environment-variables.md+++ b/docs/docs/03-configuration/01-environment-variables.md@@ -149,6 +149,7 @@ Either `OPENAI_API_KEY` or `OLLAMA_BASE_URL` need to be set for automatic taggin| CRAWLER_FULL_PAGE_ARCHIVE_MAX_SIZE_MB | No | 0 | Maximum size in MB for a stored full-page archive. Archives larger than this are discarded instead of saved, since media-rich pages can produce 1GB+ files that never render. 0 disables the limit. || CRAWLER_JOB_TIMEOUT_SEC | No | 60 | How long to wait for the crawler job to finish before timing out. If you have a slow internet connection or a low powered device, you might want to bump this up a bit || CRAWLER_NAVIGATE_TIMEOUT_SEC | No | 30 | How long to spend navigating to the page (along with its redirects). Increase this if you have a slow internet connection |+| CRAWLER_CHALLENGE_WAIT_SEC | No | 15 | When a page turns out to be a bot-protection challenge that clears by itself (e.g. Cloudflare's "Just a moment..." page), how long to wait for it to clear before capturing the page. Set to `0` to disable. || CRAWLER_BROWSER_TIMEZONE | No | Not set | IANA timezone (e.g. `America/New_York`) that the crawler's browser reports to websites. Bot-protection services flag browsers whose timezone doesn't match their IP's location, so if you crawl through a proxy, set this to the proxy's location. When not set, the browser uses its own timezone (usually UTC in containers). || CRAWLER_PREFLIGHT_USER_AGENT | No | Not set | The `User-Agent` header to send on the crawler's preflight request (the plain HTTP request that determines a link's content type and extracts early preview metadata before the full crawl). If not set, no `User-Agent` header is sent. || CRAWLER_PARSE_TIMEOUT_SEC | No | 60 | How long to wait for the HTML parsing subprocess (metadata extraction + readability) to finish before timing out. |diff --git a/packages/shared-server/src/tracingTypes.ts b/packages/shared-server/src/tracingTypes.tsindex d5581ed5..db88805d 100644--- a/packages/shared-server/src/tracingTypes.ts+++ b/packages/shared-server/src/tracingTypes.ts@@ -33,6 +33,8 @@ export type TracingAttributeKey =| "crawler.cleanup.hasPage"| "crawler.cleanup.pageClosed"| "crawler.cleanup.contextClosed"+ | "crawler.challenge.cleared"+ | "crawler.challenge.waitMs"// Database attributes| "db.system"| "db.statement"diff --git a/packages/shared/config.ts b/packages/shared/config.tsindex 11408952..d1fb2402 100644--- a/packages/shared/config.ts+++ b/packages/shared/config.ts@@ -126,6 +126,7 @@ const allEnv = z.object({BROWSER_COOKIE_PATH: z.string().optional(),CRAWLER_JOB_TIMEOUT_SEC: z.coerce.number().default(60),CRAWLER_NAVIGATE_TIMEOUT_SEC: z.coerce.number().default(30),+ CRAWLER_CHALLENGE_WAIT_SEC: z.coerce.number().default(15),CRAWLER_BROWSER_TIMEZONE: z.string().refine(@@ -420,6 +421,7 @@ const serverConfigSchema = allEnv.transform((val, ctx) => {browserCookiePath: val.BROWSER_COOKIE_PATH,jobTimeoutSec: val.CRAWLER_JOB_TIMEOUT_SEC,navigateTimeoutSec: val.CRAWLER_NAVIGATE_TIMEOUT_SEC,+ challengeWaitSec: val.CRAWLER_CHALLENGE_WAIT_SEC,browserTimezone: val.CRAWLER_BROWSER_TIMEZONE,downloadBannerImage: val.CRAWLER_DOWNLOAD_BANNER_IMAGE,storeScreenshot: val.CRAWLER_STORE_SCREENSHOT,