Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): recover crawl-finish context when FDB sheds member data (#3783)

Gergő Móriczd9a35c055a36
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 000000000..fb2ac4163
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,19 @@
+Firecrawl is a web scraper API. The directory you have access to is a monorepo:
+ - `apps/api` has the actual API and worker code
+ - `apps/*-sdk` are various SDKs
+
+When making changes to the API, here are the general steps you should take:
+1. Write some end-to-end tests that assert your win conditions, if they don't already exist
+ - 1 happy path (more is encouraged if there are multiple happy paths with significantly different code paths taken)
+ - 1+ failure path(s)
+ - Generally, E2E (called `snips` in the API) is always preferred over unit testing.
+ - In the API, always use `scrapeTimeout` from `./lib` to set the timeout you use for scrapes.
+ - These tests will be ran on a variety of configurations. You should gate tests in the following manner:
+ - If it requires fire-engine: `!process.env.TEST_SUITE_SELF_HOSTED`
+ - If it requires AI: `!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL`
+2. Write code to achieve your win conditions
+3. Run your tests using `pnpm harness jest ...`
+ - `pnpm harness` is a command that gets the API server and workers up for you to run the tests. Don't try to `pnpm start` manually.
+ - The full test suite takes a long time to run, so you should try to only execute the relevant tests locally, and let CI run the full test suite.
+4. Push to a branch, open a PR, and let CI run to verify your win condition.
+Keep these steps in mind while building your TODO list.
\ No newline at end of file
diff --git a/apps/api/package.json b/apps/api/package.json
index 8dd539c93..ff44a051f 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -174,8 +174,8 @@
"@sentry/cli",
"bigint-buffer",
"bufferutil",
- "four-flap-bigint-buffer",
"esbuild",
+ "four-flap-bigint-buffer",
"keccak",
"koffi",
"libpq",
diff --git a/apps/api/src/__tests__/nuq-fdb/core.test.ts b/apps/api/src/__tests__/nuq-fdb/core.test.ts
index b43dd5da0..591a9f9e7 100644
--- a/apps/api/src/__tests__/nuq-fdb/core.test.ts
+++ b/apps/api/src/__tests__/nuq-fdb/core.test.ts
@@ -616,6 +616,54 @@ describeIf("NuQ FDB core", () => {
await queue.jobFinish(j.id, j.lock!, null);
});
+ // A completed crawl member must retain its input data so the crawl-finish job
+ // can recover crawl-scoped context (v1, webhook, team_id) via getGroupAnyJob,
+ // matching the PG backend. Cloud sheds standalone data but not group members.
+ test("group member retains input data after completion", async () => {
+ const { queue, group } = await makeCtx("retain-groupdata");
+ const owner = freshOwner();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner);
+ const id = randomUUID();
+ await queue.addJob(
+ id,
+ scrapeData({ v1: true, team_id: owner, webhook: { url: "https://wh" } }),
+ { ownerId: owner, groupId: gid },
+ gate(10),
+ );
+
+ const [j] = await takeAll(queue, 1);
+ await queue.jobFinish(j.id, j.lock!, { ok: true });
+
+ const any = await queue.getGroupAnyJob(gid, owner);
+ expect(any?.id).toBe(id);
+ expect(any?.data).not.toBeNull();
+ expect(any?.data?.v1).toBe(true);
+ expect(any?.data?.webhook?.url).toBe("https://wh");
+ });
+
+ // ZDR members still shed input data on completion (compliance), so the
+ // crawl-finish path must tolerate null data for those crawls.
+ test("ZDR group member sheds input data on completion", async () => {
+ const { queue, group } = await makeCtx("zdr-shed");
+ const owner = freshOwner();
+ const gid = randomUUID();
+ await group.addGroup(gid, owner);
+ const id = randomUUID();
+ await queue.addJob(
+ id,
+ scrapeData({ zeroDataRetention: true }),
+ { ownerId: owner, groupId: gid },
+ gate(10),
+ );
+
+ const [j] = await takeAll(queue, 1);
+ await queue.jobFinish(j.id, j.lock!, null);
+
+ const got = await queue.getJob(id);
+ expect(got?.data).toBeNull();
+ });
+
test("removeJob releases slots and promotes backlog", async () => {
const { queue } = await makeCtx("remove");
const owner = freshOwner();
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 39b95f717..9c3dc90a1 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -163,6 +163,8 @@ export async function batchScrapeController(
createdAt: Date.now(),
maxConcurrency: req.body.maxConcurrency,
zeroDataRetention,
+ v1: true,
+ webhook: req.body.webhook,
};
if (!req.body.appendToId) {
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 92320a234..6ae0d21e7 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -143,6 +143,8 @@ export async function crawlController(
: req.body.maxConcurrency
: undefined,
zeroDataRetention,
+ v1: true,
+ webhook: req.body.webhook,
};
const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 440054e45..198e209e9 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -178,6 +178,9 @@ export async function batchScrapeController(
createdAt: Date.now(),
maxConcurrency: req.body.maxConcurrency,
zeroDataRetention,
+ v1: true,
+ webhook: req.body.webhook,
+ requestId: req.body.__agentInterop?.requestId ?? undefined,
};
if (req.body.appendToId) {
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 79d5c1497..ae7c9111a 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -202,6 +202,8 @@ export async function crawlController(
: req.body.maxConcurrency
: undefined,
zeroDataRetention,
+ v1: true,
+ webhook: req.body.webhook,
};
const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 0780a7034..f82c9feee 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -1,5 +1,6 @@
import { InternalOptions } from "../scraper/scrapeURL";
import { ScrapeOptions, TeamFlags } from "../controllers/v2/types";
+import { WebhookConfig } from "../services/webhook/types";
import { WebCrawler } from "../scraper/WebScraper/crawler";
import { redisEvictConnection } from "../services/redis";
import { logger as _logger } from "./logger";
@@ -22,6 +23,14 @@ export type StoredCrawl = {
// which queue backend this crawl's group + jobs live on; a crawl never
// spans backends. Absent on crawls created before the FDB rollout (= pg).
queueBackend?: "pg" | "fdb";
+ // Crawl-scoped context the finish path needs (completion webhook, api version,
+ // request id). Persisted here so it survives even when a member job's input
+ // data has been shed — which the FDB queue does for ZDR crawls on completion.
+ // Absent on crawls created before this was added; the finish path then falls
+ // back to reading these off a representative member's job data.
+ v1?: boolean;
+ webhook?: WebhookConfig;
+ requestId?: string;
};
export async function saveCrawl(id: string, crawl: StoredCrawl) {
diff --git a/apps/api/src/services/worker/crawl-logic.test.ts b/apps/api/src/services/worker/crawl-logic.test.ts
new file mode 100644
index 000000000..3ba506a55
--- /dev/null
+++ b/apps/api/src/services/worker/crawl-logic.test.ts
@@ -0,0 +1,97 @@
+import { finishCrawlSuper } from "./crawl-logic";
+import { getCrawl } from "../../lib/crawl-redis";
+import { logCrawl } from "../logging/log_job";
+import { createWebhookSender } from "../webhook/index";
+import type { NuQJob } from "./nuq";
+
+jest.mock("../../lib/crawl-redis", () => ({
+ finishCrawl: jest.fn(async () => {}),
+ getCrawl: jest.fn(),
+ getCrawlJobs: jest.fn(async () => []),
+ getDoneJobsOrderedLength: jest.fn(async () => 2),
+}));
+
+jest.mock("../../db/rpc", () => ({
+ creditsBilledByCrawlId: jest.fn(async () => [{ credits_billed: 0 }]),
+}));
+
+jest.mock("../../controllers/v1/crawl-status", () => ({
+ getJobs: jest.fn(async () => []),
+}));
+
+jest.mock("../logging/log_job", () => ({
+ logCrawl: jest.fn(async () => {}),
+ logBatchScrape: jest.fn(async () => {}),
+}));
+
+jest.mock("../webhook/index", () => ({
+ createWebhookSender: jest.fn(),
+ WebhookEvent: {
+ CRAWL_COMPLETED: "crawl.completed",
+ BATCH_SCRAPE_COMPLETED: "batch_scrape.completed",
+ },
+}));
+
+const baseSc = {
+ originUrl: "https://example.com",
+ crawlerOptions: {},
+ scrapeOptions: {},
+ internalOptions: { zeroDataRetention: true },
+ team_id: "team-from-sc",
+ createdAt: Date.now(),
+ zeroDataRetention: true,
+};
+
+beforeEach(() => jest.clearAllMocks());
+
+// A ZDR crawl on the FDB queue sheds the member's input data, so finishCrawlSuper
+// runs with job.data === null. It must not crash and must recover the webhook,
+// team, and api version from the stored crawl.
+test("recovers crawl context from sc when job.data is shed (ZDR)", async () => {
+ const sendMock = jest.fn();
+ (createWebhookSender as jest.Mock).mockResolvedValue({ send: sendMock });
+ (getCrawl as jest.Mock).mockResolvedValue({
+ ...baseSc,
+ v1: true,
+ webhook: { url: "https://hook.example" },
+ });
+
+ const job = {
+ id: "job-1",
+ groupId: "crawl-1",
+ ownerId: "team-from-sc",
+ data: null,
+ } as unknown as NuQJob<any>;
+
+ await expect(finishCrawlSuper(job)).resolves.not.toThrow();
+
+ expect(logCrawl).toHaveBeenCalledTimes(1);
+ expect((logCrawl as jest.Mock).mock.calls[0][0]).toMatchObject({
+ id: "crawl-1",
+ team_id: "team-from-sc",
+ request_id: "crawl-1",
+ });
+
+ expect(createWebhookSender).toHaveBeenCalledWith({
+ teamId: "team-from-sc",
+ jobId: "crawl-1",
+ webhook: { url: "https://hook.example" },
+ v0: false,
+ });
+ expect(sendMock).toHaveBeenCalledTimes(1);
+});
+
+// With no webhook on the crawl and shed data, it still completes without firing.
+test("does not crash or fire a webhook when none is configured", async () => {
+ (getCrawl as jest.Mock).mockResolvedValue({ ...baseSc, v1: true });
+
+ const job = {
+ id: "job-2",
+ groupId: "crawl-2",
+ ownerId: "team-from-sc",
+ data: null,
+ } as unknown as NuQJob<any>;
+
+ await expect(finishCrawlSuper(job)).resolves.not.toThrow();
+ expect(createWebhookSender).not.toHaveBeenCalled();
+});
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index d3ea7aa6f..ae044f172 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -34,10 +34,21 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
zeroDataRetention: sc.internalOptions.zeroDataRetention,
});
+ // On the FDB backend a completed member's input data is shed for ZDR crawls,
+ // so `job.data` can be null here. Prefer the member's job data when present,
+ // otherwise recover the crawl-scoped context persisted on the stored crawl.
+ const data = job.data;
+ const isV1 = data ? !!data.v1 : (sc.v1 ?? true);
+ const teamId = data?.team_id ?? sc.team_id;
+ const requestId = data?.requestId ?? sc.requestId ?? crawlId;
+ const zeroDataRetention = sc.zeroDataRetention || data?.zeroDataRetention;
+ const webhook = data?.webhook ?? sc.webhook;
+ const monitoring = data?.monitoring;
+
logger.info("Finishing crawl");
await finishCrawl(crawlId, logger);
- if (!job.data.v1) {
+ if (!isV1) {
const jobIDs = await getCrawlJobs(crawlId);
const jobs = (await getJobs(jobIDs)).sort(
@@ -62,19 +73,19 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
await logCrawl(
{
id: crawlId,
- request_id: job.data.requestId ?? crawlId,
+ request_id: requestId,
url: sc.originUrl!,
- team_id: job.data.team_id,
+ team_id: teamId,
options: sc.crawlerOptions,
num_docs: fullDocs.length,
credits_cost: fullDocs.reduce(
(acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),
0,
),
- zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,
+ zeroDataRetention,
cancelled: sc.cancelled ?? false,
- monitor_id: job.data.monitoring?.monitorId,
- monitor_check_id: job.data.monitoring?.checkId,
+ monitor_id: monitoring?.monitorId,
+ monitor_check_id: monitoring?.checkId,
},
false,
);
@@ -82,14 +93,14 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
await logBatchScrape(
{
id: crawlId,
- request_id: job.data.requestId ?? crawlId,
- team_id: job.data.team_id,
+ request_id: requestId,
+ team_id: teamId,
num_docs: fullDocs.length,
credits_cost: fullDocs.reduce(
(acc, doc) => acc + (doc?.metadata?.creditsUsed ?? 0),
0,
),
- zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,
+ zeroDataRetention,
cancelled: sc.cancelled ?? false,
},
false,
@@ -97,11 +108,11 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
}
// v0 web hooks, call when done with all the data
- if (!job.data.v1) {
+ if (!isV1) {
const sender = await createWebhookSender({
- teamId: job.data.team_id,
+ teamId,
jobId: crawlId,
- webhook: job.data.webhook,
+ webhook,
v0: true,
});
if (sender) {
@@ -148,16 +159,16 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
await logCrawl(
{
id: crawlId,
- request_id: job.data.requestId ?? crawlId,
+ request_id: requestId,
url: sc.originUrl!,
- team_id: job.data.team_id,
+ team_id: teamId,
options: sc.crawlerOptions,
num_docs: num_docs,
credits_cost: credits_billed ?? 0,
- zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,
+ zeroDataRetention,
cancelled: sc.cancelled ?? false,
- monitor_id: job.data.monitoring?.monitorId,
- monitor_check_id: job.data.monitoring?.checkId,
+ monitor_id: monitoring?.monitorId,
+ monitor_check_id: monitoring?.checkId,
},
false,
);
@@ -165,11 +176,11 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
await logBatchScrape(
{
id: crawlId,
- request_id: job.data.requestId ?? crawlId,
- team_id: job.data.team_id,
+ request_id: requestId,
+ team_id: teamId,
num_docs: num_docs,
credits_cost: credits_billed ?? 0,
- zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,
+ zeroDataRetention,
cancelled: sc.cancelled ?? false,
},
false,
@@ -177,11 +188,11 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
}
// v1 web hooks, call when done with no data, but with event completed
- if (job.data.v1 && job.data.webhook) {
+ if (isV1 && webhook) {
const sender = await createWebhookSender({
- teamId: job.data.team_id,
+ teamId,
jobId: crawlId,
- webhook: job.data.webhook,
+ webhook,
v0: false,
});
if (sender) {
diff --git a/apps/api/src/services/worker/nuq-fdb/queue.ts b/apps/api/src/services/worker/nuq-fdb/queue.ts
index 9d0541fe4..28b6e6cb7 100644
--- a/apps/api/src/services/worker/nuq-fdb/queue.ts
+++ b/apps/api/src/services/worker/nuq-fdb/queue.ts
@@ -810,8 +810,12 @@ export class NuQFdbQueue<JobData = any, JobReturnValue = any> {
);
}
- // shed job data early on cloud (results live in GCS) and for ZDR
- if (config.GCS_BUCKET_NAME || meta.f & F_ZDR) {
+ // Shed job input data early on cloud (results live in GCS) and always for
+ // ZDR. Group members are exempt on the plain cloud path: the crawl-finish
+ // job recovers crawl-scoped context (v1, webhook, team_id, ...) from a
+ // representative member via getGroupAnyJob, mirroring the PG backend which
+ // never sheds. ZDR still sheds even for members (compliance).
+ if (meta.f & F_ZDR || (config.GCS_BUCKET_NAME && !meta.g)) {
const r = ks.jobDataRange(id);
tn.clearRange(r.begin, r.end);
}