Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): recover crawl-finish context when FDB sheds member data (#3783)
Gergő Móriczd9a35c055a36
diff --git a/AGENTS.md b/AGENTS.mdnew file mode 100644index 000000000..fb2ac4163--- /dev/null+++ b/AGENTS.md@@ -0,0 +1,19 @@+Firecrawl is a web scraper API. The directory you have access to is a monorepo:+ - `apps/api` has the actual API and worker code+ - `apps/*-sdk` are various SDKs++When making changes to the API, here are the general steps you should take:+1. Write some end-to-end tests that assert your win conditions, if they don't already exist+ - 1 happy path (more is encouraged if there are multiple happy paths with significantly different code paths taken)+ - 1+ failure path(s)+ - Generally, E2E (called `snips` in the API) is always preferred over unit testing.+ - In the API, always use `scrapeTimeout` from `./lib` to set the timeout you use for scrapes.+ - These tests will be ran on a variety of configurations. You should gate tests in the following manner:+ - If it requires fire-engine: `!process.env.TEST_SUITE_SELF_HOSTED`+ - If it requires AI: `!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL`+2. Write code to achieve your win conditions+3. Run your tests using `pnpm harness jest ...`+ - `pnpm harness` is a command that gets the API server and workers up for you to run the tests. Don't try to `pnpm start` manually.+ - The full test suite takes a long time to run, so you should try to only execute the relevant tests locally, and let CI run the full test suite.+4. Push to a branch, open a PR, and let CI run to verify your win condition.+Keep these steps in mind while building your TODO list.\ No newline at end of filediff --git a/apps/api/package.json b/apps/api/package.jsonindex 8dd539c93..ff44a051f 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -174,8 +174,8 @@"@sentry/cli","bigint-buffer","bufferutil",- "four-flap-bigint-buffer","esbuild",+ "four-flap-bigint-buffer","keccak","koffi","libpq",diff --git a/apps/api/src/__tests__/nuq-fdb/core.test.ts b/apps/api/src/__tests__/nuq-fdb/core.test.tsindex b43dd5da0..591a9f9e7 100644--- a/apps/api/src/__tests__/nuq-fdb/core.test.ts+++ b/apps/api/src/__tests__/nuq-fdb/core.test.ts@@ -616,6 +616,54 @@ describeIf("NuQ FDB core", () => {await queue.jobFinish(j.id, j.lock!, null);});+ // A completed crawl member must retain its input data so the crawl-finish job+ // can recover crawl-scoped context (v1, webhook, team_id) via getGroupAnyJob,+ // matching the PG backend. Cloud sheds standalone data but not group members.+ test("group member retains input data after completion", async () => {+ const { queue, group } = await makeCtx("retain-groupdata");+ const owner = freshOwner();+ const gid = randomUUID();+ await group.addGroup(gid, owner);+ const id = randomUUID();+ await queue.addJob(+ id,+ scrapeData({ v1: true, team_id: owner, webhook: { url: "https://wh" } }),+ { ownerId: owner, groupId: gid },+ gate(10),+ );++ const [j] = await takeAll(queue, 1);+ await queue.jobFinish(j.id, j.lock!, { ok: true });++ const any = await queue.getGroupAnyJob(gid, owner);+ expect(any?.id).toBe(id);+ expect(any?.data).not.toBeNull();+ expect(any?.data?.v1).toBe(true);+ expect(any?.data?.webhook?.url).toBe("https://wh");+ });++ // ZDR members still shed input data on completion (compliance), so the+ // crawl-finish path must tolerate null data for those crawls.+ test("ZDR group member sheds input data on completion", async () => {+ const { queue, group } = await makeCtx("zdr-shed");+ const owner = freshOwner();+ const gid = randomUUID();+ await group.addGroup(gid, owner);+ const id = randomUUID();+ await queue.addJob(+ id,+ scrapeData({ zeroDataRetention: true }),+ { ownerId: owner, groupId: gid },+ gate(10),+ );++ const [j] = await takeAll(queue, 1);+ await queue.jobFinish(j.id, j.lock!, null);++ const got = await queue.getJob(id);+ expect(got?.data).toBeNull();+ });+test("removeJob releases slots and promotes backlog", async () => {const { queue } = await makeCtx("remove");const owner = freshOwner();diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 39b95f717..9c3dc90a1 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -163,6 +163,8 @@ export async function batchScrapeController(createdAt: Date.now(),maxConcurrency: req.body.maxConcurrency,zeroDataRetention,+ v1: true,+ webhook: req.body.webhook,};if (!req.body.appendToId) {diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 92320a234..6ae0d21e7 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -143,6 +143,8 @@ export async function crawlController(: req.body.maxConcurrency: undefined,zeroDataRetention,+ v1: true,+ webhook: req.body.webhook,};const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 440054e45..198e209e9 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -178,6 +178,9 @@ export async function batchScrapeController(createdAt: Date.now(),maxConcurrency: req.body.maxConcurrency,zeroDataRetention,+ v1: true,+ webhook: req.body.webhook,+ requestId: req.body.__agentInterop?.requestId ?? undefined,};if (req.body.appendToId) {diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 79d5c1497..ae7c9111a 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -202,6 +202,8 @@ export async function crawlController(: req.body.maxConcurrency: undefined,zeroDataRetention,+ v1: true,+ webhook: req.body.webhook,};const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 0780a7034..f82c9feee 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -1,5 +1,6 @@import { InternalOptions } from "../scraper/scrapeURL";import { ScrapeOptions, TeamFlags } from "../controllers/v2/types";+import { WebhookConfig } from "../services/webhook/types";import { WebCrawler } from "../scraper/WebScraper/crawler";import { redisEvictConnection } from "../services/redis";import { logger as _logger } from "./logger";@@ -22,6 +23,14 @@ export type StoredCrawl = {// which queue backend this crawl's group + jobs live on; a crawl never// spans backends. Absent on crawls created before the FDB rollout (= pg).queueBackend?: "pg" | "fdb";+ // Crawl-scoped context the finish path needs (completion webhook, api version,+ // request id). Persisted here so it survives even when a member job's input+ // data has been shed — which the FDB queue does for ZDR crawls on completion.+ // Absent on crawls created before this was added; the finish path then falls+ // back to reading these off a representative member's job data.+ v1?: boolean;+ webhook?: WebhookConfig;+ requestId?: string;};export async function saveCrawl(id: string, crawl: StoredCrawl) {diff --git a/apps/api/src/services/worker/crawl-logic.test.ts b/apps/api/src/services/worker/crawl-logic.test.tsnew file mode 100644index 000000000..3ba506a55--- /dev/null+++ b/apps/api/src/services/worker/crawl-logic.test.ts@@ -0,0 +1,97 @@+import { finishCrawlSuper } from "./crawl-logic";+import { getCrawl } from "../../lib/crawl-redis";+import { logCrawl } from "../logging/log_job";+import { createWebhookSender } from "../webhook/index";+import type { NuQJob } from "./nuq";++jest.mock("../../lib/crawl-redis", () => ({+ finishCrawl: jest.fn(async () => {}),+ getCrawl: jest.fn(),+ getCrawlJobs: jest.fn(async () => []),+ getDoneJobsOrderedLength: jest.fn(async () => 2),+}));++jest.mock("../../db/rpc", () => ({+ creditsBilledByCrawlId: jest.fn(async () => [{ credits_billed: 0 }]),+}));++jest.mock("../../controllers/v1/crawl-status", () => ({+ getJobs: jest.fn(async () => []),+}));++jest.mock("../logging/log_job", () => ({+ logCrawl: jest.fn(async () => {}),+ logBatchScrape: jest.fn(async () => {}),+}));++jest.mock("../webhook/index", () => ({+ createWebhookSender: jest.fn(),+ WebhookEvent: {+ CRAWL_COMPLETED: "crawl.completed",+ BATCH_SCRAPE_COMPLETED: "batch_scrape.completed",+ },+}));++const baseSc = {+ originUrl: "https://example.com",+ crawlerOptions: {},+ scrapeOptions: {},+ internalOptions: { zeroDataRetention: true },+ team_id: "team-from-sc",+ createdAt: Date.now(),+ zeroDataRetention: true,+};++beforeEach(() => jest.clearAllMocks());++// A ZDR crawl on the FDB queue sheds the member's input data, so finishCrawlSuper+// runs with job.data === null. It must not crash and must recover the webhook,+// team, and api version from the stored crawl.+test("recovers crawl context from sc when job.data is shed (ZDR)", async () => {+ const sendMock = jest.fn();+ (createWebhookSender as jest.Mock).mockResolvedValue({ send: sendMock });+ (getCrawl as jest.Mock).mockResolvedValue({+ ...baseSc,+ v1: true,+ webhook: { url: "https://hook.example" },+ });++ const job = {+ id: "job-1",+ groupId: "crawl-1",+ ownerId: "team-from-sc",+ data: null,+ } as unknown as NuQJob<any>;++ await expect(finishCrawlSuper(job)).resolves.not.toThrow();++ expect(logCrawl).toHaveBeenCalledTimes(1);+ expect((logCrawl as jest.Mock).mock.calls[0][0]).toMatchObject({+ id: "crawl-1",+ team_id: "team-from-sc",+ request_id: "crawl-1",+ });++ expect(createWebhookSender).toHaveBeenCalledWith({+ teamId: "team-from-sc",+ jobId: "crawl-1",+ webhook: { url: "https://hook.example" },+ v0: false,+ });+ expect(sendMock).toHaveBeenCalledTimes(1);+});++// With no webhook on the crawl and shed data, it still completes without firing.+test("does not crash or fire a webhook when none is configured", async () => {+ (getCrawl as jest.Mock).mockResolvedValue({ ...baseSc, v1: true });++ const job = {+ id: "job-2",+ groupId: "crawl-2",+ ownerId: "team-from-sc",+ data: null,+ } as unknown as NuQJob<any>;++ await expect(finishCrawlSuper(job)).resolves.not.toThrow();+ expect(createWebhookSender).not.toHaveBeenCalled();+});diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex d3ea7aa6f..ae044f172 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -34,10 +34,21 @@ export async function finishCrawlSuper(job: NuQJob<any>) {zeroDataRetention: sc.internalOptions.zeroDataRetention,});+ // On the FDB backend a completed member's input data is shed for ZDR crawls,+ // so `job.data` can be null here. Prefer the member's job data when present,+ // otherwise recover the crawl-scoped context persisted on the stored crawl.+ const data = job.data;+ const isV1 = data ? !!data.v1 : (sc.v1 ?? true);+ const teamId = data?.team_id ?? sc.team_id;+ const requestId = data?.requestId ?? sc.requestId ?? crawlId;+ const zeroDataRetention = sc.zeroDataRetention || data?.zeroDataRetention;+ const webhook = data?.webhook ?? sc.webhook;+ const monitoring = data?.monitoring;+logger.info("Finishing crawl");await finishCrawl(crawlId, logger);- if (!job.data.v1) {+ if (!isV1) {const jobIDs = await getCrawlJobs(crawlId);const jobs = (await getJobs(jobIDs)).sort(@@ -62,19 +73,19 @@ export async function finishCrawlSuper(job: NuQJob<any>) {await logCrawl({id: crawlId,- request_id: job.data.requestId ?? crawlId,+ request_id: requestId,url: sc.originUrl!,- team_id: job.data.team_id,+ team_id: teamId,options: sc.crawlerOptions,num_docs: fullDocs.length,credits_cost: fullDocs.reduce((acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),0,),- zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,+ zeroDataRetention,cancelled: sc.cancelled ?? false,- monitor_id: job.data.monitoring?.monitorId,- monitor_check_id: job.data.monitoring?.checkId,+ monitor_id: monitoring?.monitorId,+ monitor_check_id: monitoring?.checkId,},false,);@@ -82,14 +93,14 @@ export async function finishCrawlSuper(job: NuQJob<any>) {await logBatchScrape({id: crawlId,- request_id: job.data.requestId ?? crawlId,- team_id: job.data.team_id,+ request_id: requestId,+ team_id: teamId,num_docs: fullDocs.length,credits_cost: fullDocs.reduce((acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),0,),- zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,+ zeroDataRetention,cancelled: sc.cancelled ?? false,},false,@@ -97,11 +108,11 @@ export async function finishCrawlSuper(job: NuQJob<any>) {}// v0 web hooks, call when done with all the data- if (!job.data.v1) {+ if (!isV1) {const sender = await createWebhookSender({- teamId: job.data.team_id,+ teamId,jobId: crawlId,- webhook: job.data.webhook,+ webhook,v0: true,});if (sender) {@@ -148,16 +159,16 @@ export async function finishCrawlSuper(job: NuQJob<any>) {await logCrawl({id: crawlId,- request_id: job.data.requestId ?? crawlId,+ request_id: requestId,url: sc.originUrl!,- team_id: job.data.team_id,+ team_id: teamId,options: sc.crawlerOptions,num_docs: num_docs,credits_cost: credits_billed ?? 0,- zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,+ zeroDataRetention,cancelled: sc.cancelled ?? false,- monitor_id: job.data.monitoring?.monitorId,- monitor_check_id: job.data.monitoring?.checkId,+ monitor_id: monitoring?.monitorId,+ monitor_check_id: monitoring?.checkId,},false,);@@ -165,11 +176,11 @@ export async function finishCrawlSuper(job: NuQJob<any>) {await logBatchScrape({id: crawlId,- request_id: job.data.requestId ?? crawlId,- team_id: job.data.team_id,+ request_id: requestId,+ team_id: teamId,num_docs: num_docs,credits_cost: credits_billed ?? 0,- zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,+ zeroDataRetention,cancelled: sc.cancelled ?? false,},false,@@ -177,11 +188,11 @@ export async function finishCrawlSuper(job: NuQJob<any>) {}// v1 web hooks, call when done with no data, but with event completed- if (job.data.v1 && job.data.webhook) {+ if (isV1 && webhook) {const sender = await createWebhookSender({- teamId: job.data.team_id,+ teamId,jobId: crawlId,- webhook: job.data.webhook,+ webhook,v0: false,});if (sender) {diff --git a/apps/api/src/services/worker/nuq-fdb/queue.ts b/apps/api/src/services/worker/nuq-fdb/queue.tsindex 9d0541fe4..28b6e6cb7 100644--- a/apps/api/src/services/worker/nuq-fdb/queue.ts+++ b/apps/api/src/services/worker/nuq-fdb/queue.ts@@ -810,8 +810,12 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {);}- // shed job data early on cloud (results live in GCS) and for ZDR- if (config.GCS_BUCKET_NAME || meta.f & F_ZDR) {+ // Shed job input data early on cloud (results live in GCS) and always for+ // ZDR. Group members are exempt on the plain cloud path: the crawl-finish+ // job recovers crawl-scoped context (v1, webhook, team_id, ...) from a+ // representative member via getGroupAnyJob, mirroring the PG backend which+ // never sheds. ZDR still sheds even for members (compliance).+ if (meta.f & F_ZDR || (config.GCS_BUCKET_NAME && !meta.g)) {const r = ks.jobDataRange(id);tn.clearRange(r.begin, r.end);}