Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

feat: add a script to adhocly test the crawler + improve chrome flags (#2929)

Mohamed Bassem5b0d7fe6f805
diff --git a/apps/workers/scripts/crawlAdhoc.ts b/apps/workers/scripts/crawlAdhoc.ts
new file mode 100644
index 00000000..b322e9f3
--- /dev/null
+++ b/apps/workers/scripts/crawlAdhoc.ts
@@ -0,0 +1,212 @@
+import "dotenv/config";
+
+import {
+ appendFileSync,
+ mkdirSync,
+ mkdtempSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { parseArgs } from "node:util";
+
+/**
+ * Adhoc crawl CLI — drives the REAL crawlPage() browser-fetch path without the
+ * queue, DB job, or downstream (inference/search/archival) machinery. Meant for
+ * manual debugging and for the crawler A/B test (021-crawler-stealth-ab-test).
+ *
+ * DB safety: crawlPage() with the override does NOT read or write any app data
+ * (the per-user lookup is skipped; asset/bookmark/search/webhook writes live in
+ * downstream functions this CLI never calls). BUT importing the crawler module
+ * transitively opens a SQLite connection (packages/db/drizzle.ts) against
+ * DATA_DIR. To guarantee we never touch the real DB, this bootstrap repoints
+ * DATA_DIR at a throwaway temp dir BEFORE any app module is imported. Set
+ * KARAKEEP_ADHOC_USE_REAL_DB=1 to opt out and use the configured DB instead.
+ *
+ * The two anti-detection levers are controlled purely by env, so the test
+ * driver can flip them per attempt:
+ * BROWSER_WEB_URL -> which Chrome container (flags arm)
+ * REBROWSER_PATCHES_RUNTIME_FIX_MODE -> "0" disables the rebrowser patch
+ *
+ * Usage:
+ * BROWSER_WEB_URL=http://127.0.0.1:9222 \
+ * tsx scripts/crawlAdhoc.ts --url https://example.com --repeats 3 \
+ * --arm baseline --out results.jsonl
+ * (use 127.0.0.1, not localhost — Chrome's DevTools host check rejects [::1])
+ */
+
+// Repoint the DB at a throwaway location before importing anything that pulls
+// in @karakeep/db. Static imports above are node built-ins + dotenv only.
+if (process.env.KARAKEEP_ADHOC_USE_REAL_DB !== "1") {
+ const scratch = mkdtempSync(path.join(os.tmpdir(), "karakeep-adhoc-"));
+ process.env.DATA_DIR = scratch;
+ process.env.ASSETS_DIR = path.join(scratch, "assets");
+ mkdirSync(process.env.ASSETS_DIR, { recursive: true });
+ // Don't leak a throwaway dir per run. Must be synchronous — "exit" handlers
+ // can't await, so an async fs call here would never complete before exit.
+ process.on("exit", () => rmSync(scratch, { recursive: true, force: true }));
+}
+
+// Interstitial-only markers: strings that appear on an actual challenge/block
+// PAGE, not merely in the bot-management scripts (e.g. /cdn-cgi/challenge-platform,
+// px-captcha, _px3) that Cloudflare/PerimeterX inject on normal pages too. Using
+// those loose tokens false-positives real content as blocked. Matched
+// case-insensitively; only trusted when the page is also small (see scoreHtml).
+const BLOCK_MARKERS = [
+ "just a moment",
+ "checking your browser",
+ "attention required! | cloudflare",
+ "enable javascript and cookies to continue",
+ "verify you are human",
+ "geo.captcha-delivery.com",
+ "please verify you are a human",
+];
+
+// A challenge interstitial is small; real content pages are large. Above this
+// size we trust a 200 as real even if a marker substring happens to appear.
+const INTERSTITIAL_MAX_BYTES = 100_000;
+
+function scoreHtml(
+ statusCode: number,
+ html: string,
+): { blocked: boolean; marker: string | null } {
+ // Status is the primary, low-false-positive signal.
+ if (statusCode === 403 || statusCode === 429 || statusCode === 503) {
+ const lower = html.toLowerCase();
+ return {
+ blocked: true,
+ marker: BLOCK_MARKERS.find((m) => lower.includes(m)) ?? "status",
+ };
+ }
+ const lower = html.toLowerCase();
+ const marker = BLOCK_MARKERS.find((m) => lower.includes(m)) ?? null;
+ // Only treat a 2xx as blocked if it looks like a small interstitial.
+ const blocked = marker !== null && html.length < INTERSTITIAL_MAX_BYTES;
+ return { blocked, marker: blocked ? marker : null };
+}
+
+async function main() {
+ // Dynamic imports so the DATA_DIR override above takes effect first.
+ const { default: serverConfig } = await import("@karakeep/shared/config");
+ const { selectRunProxies } = await import("network");
+ const { crawlPage, CrawlerWorker } = await import("workers/crawlerWorker");
+
+ const { values } = parseArgs({
+ options: {
+ url: { type: "string", multiple: true },
+ repeats: { type: "string", default: "1" },
+ arm: { type: "string", default: "unlabeled" },
+ out: { type: "string" },
+ pdf: { type: "boolean", default: false },
+ "screenshot-dir": { type: "string" },
+ "timeout-sec": { type: "string", default: "60" },
+ },
+ });
+
+ const urls = values.url ?? [];
+ if (urls.length === 0) {
+ console.error(
+ "No --url provided. Example: tsx scripts/crawlAdhoc.ts --url https://example.com",
+ );
+ process.exit(2);
+ }
+ const repeats = Number(values.repeats);
+ const timeoutMs = Number(values["timeout-sec"]) * 1000;
+ const screenshotDir = values["screenshot-dir"];
+ if (screenshotDir) {
+ mkdirSync(screenshotDir, { recursive: true });
+ }
+
+ console.error(
+ `[adhoc] arm=${values.arm} browser=${serverConfig.crawler.browserWebUrl ?? "(browserless)"} ` +
+ `rebrowserFix=${process.env.REBROWSER_PATCHES_RUNTIME_FIX_MODE === "0" ? "off" : "on"} ` +
+ `db=${process.env.KARAKEEP_ADHOC_USE_REAL_DB === "1" ? "REAL" : "scratch"}`,
+ );
+
+ await CrawlerWorker.prepareForAdhoc();
+
+ const runProxy = selectRunProxies();
+ const lines: string[] = [];
+
+ for (const url of urls) {
+ // Slug used for both the jobId (so tracing/logs stay distinct per URL) and
+ // the screenshot filename.
+ const safeUrl = url.replace(/[^a-z0-9]+/gi, "_").slice(0, 80);
+ for (let rep = 0; rep < repeats; rep++) {
+ const jobId = `adhoc-${values.arm}-${safeUrl}-${rep}`;
+ const abort = new AbortController();
+ const timer = setTimeout(() => abort.abort(), timeoutMs);
+ const startedAt = Date.now();
+ let record: Record<string, unknown>;
+ try {
+ const res = await crawlPage(
+ jobId,
+ url,
+ "adhoc",
+ values.pdf,
+ abort.signal,
+ runProxy,
+ // Force the full browser path and skip the per-user DB lookup.
+ true,
+ );
+ const { blocked, marker } = scoreHtml(res.statusCode, res.htmlContent);
+ if (screenshotDir && res.screenshot) {
+ writeFileSync(
+ path.join(screenshotDir, `${values.arm}-${safeUrl}-${rep}.jpg`),
+ res.screenshot,
+ );
+ }
+ record = {
+ arm: values.arm,
+ url,
+ finalUrl: res.url,
+ rep,
+ status: res.statusCode,
+ blocked,
+ marker,
+ htmlLen: res.htmlContent.length,
+ hasScreenshot: res.screenshot !== undefined,
+ hasPdf: res.pdf !== undefined,
+ latencyMs: Date.now() - startedAt,
+ error: null,
+ };
+ } catch (e) {
+ record = {
+ arm: values.arm,
+ url,
+ finalUrl: null,
+ rep,
+ status: null,
+ blocked: null,
+ marker: null,
+ htmlLen: 0,
+ hasScreenshot: false,
+ hasPdf: false,
+ latencyMs: Date.now() - startedAt,
+ error: e instanceof Error ? e.message : String(e),
+ };
+ } finally {
+ clearTimeout(timer);
+ }
+ const line = JSON.stringify(record);
+ lines.push(line);
+ console.error(
+ `[adhoc] ${values.arm} ${url} rep=${rep} -> status=${record.status} blocked=${record.blocked} marker=${record.marker ?? "-"} htmlLen=${record.htmlLen}`,
+ );
+ }
+ }
+
+ const output = lines.join("\n") + "\n";
+ if (values.out) {
+ // Append so a driver can interleave many single-attempt invocations.
+ appendFileSync(values.out, output);
+ } else {
+ process.stdout.write(output);
+ }
+
+ // crawlPage leaves the shared browser connection open; exit explicitly.
+ process.exit(0);
+}
+
+void main();
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index 91d1fff9..933ce1d3 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -369,6 +369,38 @@ export class CrawlerWorker {
return CrawlerWorker.initPromise;
}
+ // Boots the browser/adblocker/cookies without the queue runner. Used by the
+ // adhoc crawl CLI (scripts/crawlAdhoc.ts).
+ static async prepareForAdhoc(): Promise<void> {
+ await CrawlerWorker.ensureInitialized();
+
+ // The adhoc CLI exists to exercise the REAL browser path. When no browser is
+ // reachable, crawlPage() silently falls back to a plain HTTP fetch
+ // (browserlessCrawlPage), which would quietly corrupt A/B results with a run
+ // that never executed JS/screenshots. Fail loudly instead. Two cases where
+ // the fallback is silent: no browser is configured at all, and a
+ // non-on-demand connection that failed to establish at init (globalBrowser
+ // stays undefined). On-demand connections throw per-crawl, so those surface
+ // as visible errors already.
+ const hasBrowserBackend =
+ !!serverConfig.crawler.browserWebUrl ||
+ !!serverConfig.crawler.browserWebSocketUrl;
+ if (!hasBrowserBackend) {
+ throw new Error(
+ "[adhoc] No browser backend configured — refusing to run. crawlPage() " +
+ "would silently fall back to a plain HTTP fetch. Set BROWSER_WEB_URL " +
+ "or BROWSER_WEBSOCKET_URL to a reachable Chrome.",
+ );
+ }
+ if (!serverConfig.crawler.browserConnectOnDemand && !globalBrowser) {
+ throw new Error(
+ "[adhoc] Browser failed to connect — refusing to run. crawlPage() would " +
+ "silently fall back to a plain HTTP fetch. Check that BROWSER_WEB_URL / " +
+ "BROWSER_WEBSOCKET_URL points at a reachable Chrome.",
+ );
+ }
+ }
+
static async build(queue: Queue<ZCrawlLinkRequest>) {
await CrawlerWorker.ensureInitialized();
@@ -531,13 +563,16 @@ async function browserlessCrawlPage(
);
}
-async function crawlPage(
+export async function crawlPage(
jobId: string,
url: string,
userId: string,
forceStorePdf: boolean,
abortSignal: AbortSignal,
runProxy: RunProxyConfig,
+ // When set, skips the per-user browserCrawlingEnabled DB lookup and uses this
+ // value instead. Used by the adhoc crawl CLI, which has no user row.
+ browserCrawlingEnabledOverride?: boolean,
): Promise<{
htmlContent: string;
screenshot: Buffer | undefined;
@@ -558,17 +593,21 @@ async function crawlPage(
},
},
async () => {
- const userData = await db.query.users.findFirst({
- where: eq(users.id, userId),
- columns: { browserCrawlingEnabled: true },
- });
- if (!userData) {
- logger.error(`[Crawler][${jobId}] User ${userId} not found`);
- throw new Error(`User ${userId} not found`);
+ let browserCrawlingEnabled: boolean | null;
+ if (browserCrawlingEnabledOverride !== undefined) {
+ browserCrawlingEnabled = browserCrawlingEnabledOverride;
+ } else {
+ const userData = await db.query.users.findFirst({
+ where: eq(users.id, userId),
+ columns: { browserCrawlingEnabled: true },
+ });
+ if (!userData) {
+ logger.error(`[Crawler][${jobId}] User ${userId} not found`);
+ throw new Error(`User ${userId} not found`);
+ }
+ browserCrawlingEnabled = userData.browserCrawlingEnabled;
}
- const browserCrawlingEnabled = userData.browserCrawlingEnabled;
-
if (browserCrawlingEnabled !== null && !browserCrawlingEnabled) {
return browserlessCrawlPage(jobId, url, abortSignal, runProxy);
}
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index ad4c06b6..93286fc6 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -23,12 +23,15 @@ services:
image: gcr.io/zenika-hub/alpine-chrome:124
restart: unless-stopped
command:
+ - --headless=new
- --no-sandbox
- --disable-gpu
- --disable-dev-shm-usage
- --remote-debugging-address=0.0.0.0
- --remote-debugging-port=9222
- --hide-scrollbars
+ - --disable-blink-features=AutomationControlled
+ - --window-size=1440,900
meilisearch:
image: getmeili/meilisearch:v1.41.0
restart: unless-stopped