Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

feat(workers): wait for self-clearing bot challenges before capturing

Mohamed Bassem18003460c6fa
diff --git a/apps/workers/workers/crawler/challenge.ts b/apps/workers/workers/crawler/challenge.ts
new file mode 100644
index 00000000..939b9844
--- /dev/null
+++ b/apps/workers/workers/crawler/challenge.ts
@@ -0,0 +1,97 @@
+// Waits out non-interactive bot-protection challenges (e.g. Cloudflare's
+// "Just a moment..." page or DataDome's device check). These interstitials run
+// some JS and then reload into the real page on their own when the browser
+// passes. Capturing right after the initial load would store the challenge
+// page instead, so when one is detected we give it a bounded amount of time to
+// clear.
+import type { Page } from "patchright";
+import { abortRace, raceWith } from "utils";
+
+import { setSpanAttributes } from "@karakeep/shared-server";
+import serverConfig from "@karakeep/shared/config";
+import logger from "@karakeep/shared/logger";
+
+import {
+ isLikelyChallengePage,
+ isWaitableChallenge,
+} from "../utils/metadataResolver";
+
+// Statuses bot-protection vendors serve their interstitials with.
+const CHALLENGE_STATUS_CODES = new Set([401, 403, 429, 503]);
+
+const POLL_INTERVAL_MS = 500;
+const SETTLE_TIMEOUT_MS = 5000;
+
+async function pageIsWaitableChallenge(page: Page): Promise<boolean> {
+ try {
+ const [title, htmlContent] = await Promise.all([
+ page.title(),
+ page.content(),
+ ]);
+ return isWaitableChallenge(title, htmlContent);
+ } catch {
+ // The page is navigating (typically the challenge reloading into the real
+ // page); treat it as still pending and check again on the next poll.
+ return true;
+ }
+}
+
+/**
+ * If the page looks like a self-clearing bot challenge, waits (up to
+ * CRAWLER_CHALLENGE_WAIT_SEC) for it to go away and for the resulting page to
+ * settle. Returns without waiting for regular pages.
+ */
+export async function waitForChallengeToClear(
+ page: Page,
+ statusCode: number,
+ jobId: string,
+ abortSignal: AbortSignal,
+): Promise<void> {
+ const waitMs = serverConfig.crawler.challengeWaitSec * 1000;
+ if (waitMs <= 0) {
+ return;
+ }
+ // Cheap gate so regular pages don't pay for serializing their DOM.
+ const title = await page.title().catch(() => "");
+ if (
+ !CHALLENGE_STATUS_CODES.has(statusCode) &&
+ !isLikelyChallengePage({ title })
+ ) {
+ return;
+ }
+ if (!(await pageIsWaitableChallenge(page))) {
+ return;
+ }
+
+ logger.info(
+ `[Crawler][${jobId}] The page looks like a bot challenge, waiting up to ${serverConfig.crawler.challengeWaitSec}s for it to clear`,
+ );
+ const startedAt = Date.now();
+ const deadline = startedAt + waitMs;
+ let cleared = false;
+ while (Date.now() < deadline) {
+ await raceWith<unknown>(
+ new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)),
+ abortRace(abortSignal),
+ );
+ if (!(await pageIsWaitableChallenge(page))) {
+ cleared = true;
+ break;
+ }
+ }
+
+ if (cleared) {
+ await page
+ .waitForLoadState("networkidle", { timeout: SETTLE_TIMEOUT_MS })
+ .catch(() => undefined);
+ }
+ abortSignal.throwIfAborted();
+ const waitedMs = Date.now() - startedAt;
+ setSpanAttributes({
+ "crawler.challenge.cleared": cleared,
+ "crawler.challenge.waitMs": waitedMs,
+ });
+ logger.info(
+ `[Crawler][${jobId}] Bot challenge ${cleared ? "cleared" : "did not clear"} after ${waitedMs}ms`,
+ );
+}
diff --git a/apps/workers/workers/crawler/crawlPage.ts b/apps/workers/workers/crawler/crawlPage.ts
index 93522d25..82692ea6 100644
--- a/apps/workers/workers/crawler/crawlPage.ts
+++ b/apps/workers/workers/crawler/crawlPage.ts
@@ -47,6 +47,7 @@ import {
trackContext,
untrackContext,
} from "./browser";
+import { waitForChallengeToClear } from "./challenge";
import { truncateUrl } from "./utils";
const tracer = getTracer("@karakeep/workers");
@@ -695,6 +696,18 @@ export async function crawlPage(
);
}
const targetUrl = navigationValidation.url.toString();
+ // Tracks the status of the latest main-frame document, which differs
+ // from the initial navigation's when a bot challenge clears and
+ // reloads into the real page.
+ let mainDocumentStatus: number | undefined;
+ activePage.on("response", (res) => {
+ if (
+ res.request().isNavigationRequest() &&
+ res.frame() === activePage.mainFrame()
+ ) {
+ mainDocumentStatus = res.status();
+ }
+ });
logger.info(`[Crawler][${jobId}] Navigating to "${targetUrl}"`);
const response = await withSpan(
tracer,
@@ -751,9 +764,21 @@ export async function crawlPage(
setup.autoconsentEnabled,
abortSignal,
);
-
abortSignal.throwIfAborted();
+ await withSpan(
+ tracer,
+ "crawlerWorker.crawlPage.waitForChallenge",
+ { attributes: { "job.id": jobId } },
+ () =>
+ waitForChallengeToClear(
+ activePage,
+ mainDocumentStatus ?? response?.status() ?? 0,
+ jobId,
+ abortSignal,
+ ),
+ );
+
logger.info(
`[Crawler][${jobId}] Finished waiting for the page to load.`,
);
@@ -767,7 +792,7 @@ export async function crawlPage(
return {
htmlContent,
- statusCode: response?.status() ?? 0,
+ statusCode: mainDocumentStatus ?? response?.status() ?? 0,
screenshot,
pdf,
url: activePage.url(),
diff --git a/apps/workers/workers/utils/metadataResolver.test.ts b/apps/workers/workers/utils/metadataResolver.test.ts
index 6cce8021..0801ccec 100644
--- a/apps/workers/workers/utils/metadataResolver.test.ts
+++ b/apps/workers/workers/utils/metadataResolver.test.ts
@@ -1,6 +1,10 @@
import { describe, expect, it } from "vitest";
-import { isLikelyChallengePage, resolveMetadata } from "./metadataResolver";
+import {
+ isLikelyChallengePage,
+ isWaitableChallenge,
+ resolveMetadata,
+} from "./metadataResolver";
function meta(fields: Record<string, string | null | undefined>) {
return {
@@ -139,3 +143,41 @@ describe("isLikelyChallengePage", () => {
);
});
});
+
+describe("isWaitableChallenge", () => {
+ it("waits on Cloudflare's interstitial", () => {
+ expect(
+ isWaitableChallenge(
+ "Just a moment...",
+ "<html><head><title>Just a moment...</title></head><body><script>window._cf_chl_opt={}</script></body></html>",
+ ),
+ ).toBe(true);
+ });
+
+ it("waits on DataDome's device check", () => {
+ expect(
+ isWaitableChallenge(
+ "wsj.com",
+ "<html><body><script>var dd={'rt':'i','cid':'x','host':'geo.captcha-delivery.com'}</script></body></html>",
+ ),
+ ).toBe(true);
+ });
+
+ it("doesn't wait on DataDome's captcha, which needs a human", () => {
+ expect(
+ isWaitableChallenge(
+ "wsj.com",
+ "<html><body><script>var dd={'rt':'c','cid':'x','host':'geo.captcha-delivery.com'}</script></body></html>",
+ ),
+ ).toBe(false);
+ });
+
+ it("doesn't wait on regular pages", () => {
+ expect(
+ isWaitableChallenge(
+ "Example Domain",
+ "<html><head><title>Example Domain</title></head><body><p>Hello</p></body></html>",
+ ),
+ ).toBe(false);
+ });
+});
diff --git a/apps/workers/workers/utils/metadataResolver.ts b/apps/workers/workers/utils/metadataResolver.ts
index 35b9b300..37cea2f3 100644
--- a/apps/workers/workers/utils/metadataResolver.ts
+++ b/apps/workers/workers/utils/metadataResolver.ts
@@ -82,6 +82,24 @@ export function isLikelyChallengePage({
return CHALLENGE_PAGE_BODY_MARKERS.some((marker) => lowered.includes(marker));
}
+// DataDome's captcha page (as opposed to its self-clearing device check) needs
+// a human to solve it, so there's no point waiting for it to clear.
+const DATADOME_HARD_CAPTCHA = /['"]rt['"]\s*:\s*['"]c['"]/;
+
+/**
+ * Whether the page is a bot challenge that may clear by itself (and reload
+ * into the real page) if the browser keeps waiting.
+ */
+export function isWaitableChallenge(
+ title: string,
+ htmlContent: string,
+): boolean {
+ return (
+ isLikelyChallengePage({ title, htmlContent }) &&
+ !DATADOME_HARD_CAPTCHA.test(htmlContent)
+ );
+}
+
/**
* Merges the browser render's metadata with the preflight probe's, per field.
* The render normally wins (it sees the post-JS DOM), with the probe filling
diff --git a/docs/docs/03-configuration/01-environment-variables.md b/docs/docs/03-configuration/01-environment-variables.md
index d2297b0d..bec66794 100644
--- a/docs/docs/03-configuration/01-environment-variables.md
+++ b/docs/docs/03-configuration/01-environment-variables.md
@@ -149,6 +149,7 @@ Either `OPENAI_API_KEY` or `OLLAMA_BASE_URL` need to be set for automatic taggin
| CRAWLER_FULL_PAGE_ARCHIVE_MAX_SIZE_MB | No | 0 | Maximum size in MB for a stored full-page archive. Archives larger than this are discarded instead of saved, since media-rich pages can produce 1GB+ files that never render. 0 disables the limit. |
| CRAWLER_JOB_TIMEOUT_SEC | No | 60 | How long to wait for the crawler job to finish before timing out. If you have a slow internet connection or a low powered device, you might want to bump this up a bit |
| CRAWLER_NAVIGATE_TIMEOUT_SEC | No | 30 | How long to spend navigating to the page (along with its redirects). Increase this if you have a slow internet connection |
+| CRAWLER_CHALLENGE_WAIT_SEC | No | 15 | When a page turns out to be a bot-protection challenge that clears by itself (e.g. Cloudflare's "Just a moment..." page), how long to wait for it to clear before capturing the page. Set to `0` to disable. |
| CRAWLER_BROWSER_TIMEZONE | No | Not set | IANA timezone (e.g. `America/New_York`) that the crawler's browser reports to websites. Bot-protection services flag browsers whose timezone doesn't match their IP's location, so if you crawl through a proxy, set this to the proxy's location. When not set, the browser uses its own timezone (usually UTC in containers). |
| CRAWLER_PREFLIGHT_USER_AGENT | No | Not set | The `User-Agent` header to send on the crawler's preflight request (the plain HTTP request that determines a link's content type and extracts early preview metadata before the full crawl). If not set, no `User-Agent` header is sent. |
| CRAWLER_PARSE_TIMEOUT_SEC | No | 60 | How long to wait for the HTML parsing subprocess (metadata extraction + readability) to finish before timing out. |
diff --git a/packages/shared-server/src/tracingTypes.ts b/packages/shared-server/src/tracingTypes.ts
index d5581ed5..db88805d 100644
--- a/packages/shared-server/src/tracingTypes.ts
+++ b/packages/shared-server/src/tracingTypes.ts
@@ -33,6 +33,8 @@ export type TracingAttributeKey =
| "crawler.cleanup.hasPage"
| "crawler.cleanup.pageClosed"
| "crawler.cleanup.contextClosed"
+ | "crawler.challenge.cleared"
+ | "crawler.challenge.waitMs"
// Database attributes
| "db.system"
| "db.statement"
diff --git a/packages/shared/config.ts b/packages/shared/config.ts
index 11408952..d1fb2402 100644
--- a/packages/shared/config.ts
+++ b/packages/shared/config.ts
@@ -126,6 +126,7 @@ const allEnv = z.object({
BROWSER_COOKIE_PATH: z.string().optional(),
CRAWLER_JOB_TIMEOUT_SEC: z.coerce.number().default(60),
CRAWLER_NAVIGATE_TIMEOUT_SEC: z.coerce.number().default(30),
+ CRAWLER_CHALLENGE_WAIT_SEC: z.coerce.number().default(15),
CRAWLER_BROWSER_TIMEZONE: z
.string()
.refine(
@@ -420,6 +421,7 @@ const serverConfigSchema = allEnv.transform((val, ctx) => {
browserCookiePath: val.BROWSER_COOKIE_PATH,
jobTimeoutSec: val.CRAWLER_JOB_TIMEOUT_SEC,
navigateTimeoutSec: val.CRAWLER_NAVIGATE_TIMEOUT_SEC,
+ challengeWaitSec: val.CRAWLER_CHALLENGE_WAIT_SEC,
browserTimezone: val.CRAWLER_BROWSER_TIMEZONE,
downloadBannerImage: val.CRAWLER_DOWNLOAD_BANNER_IMAGE,
storeScreenshot: val.CRAWLER_STORE_SCREENSHOT,