Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): send FirePDF a per-org training decision (#5395)

Gergő Móriczc351eb6dac46
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 27f103c06..b637819a4 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -353,7 +353,7 @@ export async function batchScrapeController(
agentIndexOnly: (req as any).agentIndexOnly ?? false,
threatProtection: threatProtection.policy ?? undefined,
// Safe Mode resolves per-URL at the scrapeURL backstop from these flags.
- teamFlags: req.acuc?.flags ?? undefined,
+ teamFlags: req.acuc?.flags ?? null,
safeModeBypassed: safeMode.bypassed === true,
}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter
team_id: req.auth.team_id,
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 5f709a9d8..3c042e5d7 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -273,7 +273,7 @@ export async function crawlController(
agentIndexOnly: (req as any).agentIndexOnly ?? false,
threatProtection: threatProtection.policy ?? undefined,
// Safe Mode resolves per-URL at the scrapeURL backstop from these flags.
- teamFlags: req.acuc?.flags ?? undefined,
+ teamFlags: req.acuc?.flags ?? null,
safeModeBypassed: safeMode.bypassed === true,
}, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
team_id: req.auth.team_id,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index e259a5268..2a20a0032 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1406,6 +1406,9 @@ export type TeamFlags = {
// The org's DPA (or partner amendment) restricts how its data may be
// handled. Informational only: the API does not change behavior on it.
dpaRestricted?: boolean;
+ // The org's contract bars training on its data: FirePDF never captures its
+ // documents (buildFirePdfTrainingCapture).
+ trainingRestricted?: boolean;
organizationDataSourceAccess?: Record<
string,
{
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index d6d4bfc55..ea304cad1 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -442,7 +442,7 @@ export async function batchScrapeController(
threatProtection: threatProtection.policy ?? undefined,
// Safe Mode rides the batch payload so each URL resolves it per-URL
// at the scrapeURL backstop.
- teamFlags: req.acuc?.flags ?? undefined,
+ teamFlags: req.acuc?.flags ?? null,
safeModeBypassed: safeMode.bypassed === true,
}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter
team_id: req.auth.team_id,
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 0f24298c6..1438ae7c6 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -343,7 +343,7 @@ export async function crawlController(
threatProtection: threatProtection.policy ?? undefined,
// Safe Mode rides the crawl payload so every child scrape resolves it
// per-URL at the scrapeURL backstop (allowlist applies per child).
- teamFlags: req.acuc?.flags ?? undefined,
+ teamFlags: req.acuc?.flags ?? null,
safeModeBypassed: safeMode.bypassed === true,
},
team_id: req.auth.team_id,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index a96ed5e2c..ef554b1b1 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -2173,6 +2173,9 @@ export type TeamFlags = {
// The org's DPA (or partner amendment) restricts how its data may be
// handled. Informational only: the API does not change behavior on it.
dpaRestricted?: boolean;
+ // The org's contract bars training on its data: FirePDF never captures its
+ // documents (buildFirePdfTrainingCapture).
+ trainingRestricted?: boolean;
organizationDataSourceAccess?: Record<
string,
{
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
index 322cc52c1..e7eb4fcb4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
@@ -469,6 +469,11 @@ describe("FirePDF table format", () => {
);
it("puts the format in async job options only when set", () => {
+ const trainingCapture = {
+ eligible: true,
+ policy_ref: "firecrawl-org-training-v1",
+ decision_ref: "eligible",
+ } as const;
const base = {
maxPages: undefined,
pagesProcessed: 2,
@@ -476,12 +481,25 @@ describe("FirePDF table format", () => {
includePageMarkdown: false,
includeBlocks: false,
pageMarkers: false,
+ trainingCapture,
};
expect(buildFirePdfJobOptions({ ...base, tableFormat: "dynamic" })).toEqual(
- { pages_estimate: 2, mode: "auto", tableFormat: "dynamic" },
+ {
+ pages_estimate: 2,
+ mode: "auto",
+ tableFormat: "dynamic",
+ training_capture: trainingCapture,
+ },
);
expect(buildFirePdfJobOptions({ ...base, tableFormat: undefined })).toEqual(
- { pages_estimate: 2, mode: "auto" },
+ { pages_estimate: 2, mode: "auto", training_capture: trainingCapture },
);
+ expect(
+ buildFirePdfJobOptions({
+ ...base,
+ tableFormat: undefined,
+ trainingCapture: undefined,
+ }),
+ ).toEqual({ pages_estimate: 2, mode: "auto" });
});
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts
index 39b13a047..8dda6556b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts
@@ -10,7 +10,10 @@ import {
import { cacheKeyShape, tryGetCached } from "./cache";
import { lookupAdoptableFirePdfJob } from "./lookup";
import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "./utils";
-import { firePdfTableFormat } from "./request-metadata";
+import {
+ buildFirePdfJobTrainingCapture,
+ firePdfTableFormat,
+} from "./request-metadata";
type FirePdfByReferenceAttemptArgs = {
meta: Meta;
@@ -147,6 +150,7 @@ export async function runFirePdfByReferenceAttempt(
includeBlocks,
pageMarkers,
tableFormat: firePdfTableFormat(meta),
+ trainingCapture: buildFirePdfJobTrainingCapture(meta),
}),
);
meta.abort.throwIfAborted();
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
index ea46723f9..b2bff279c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
@@ -6,6 +6,44 @@ import { hasCustomRequestContext } from "../../../lib/request-context";
/** What we hand fire-pdf: a PDF, or a raster image wrapped as a one-page document. */
export type FirePdfSourceKind = "pdf" | "image";
+/** FirePDF's training decision; its capture lanes run only on an eligible one. */
+export type FirePdfTrainingCapture = {
+ eligible: boolean;
+ policy_ref: "firecrawl-org-training-v1";
+ decision_ref: "zdr" | "org_unresolved" | "training_restricted" | "eligible";
+};
+
+/** Fail closed: eligible only outside ZDR, for an org whose id and flags were
+ * snapshotted at acceptance and whose flags carry no training restriction. */
+export function buildFirePdfTrainingCapture(
+ meta: Meta,
+): FirePdfTrainingCapture {
+ const decision = (
+ decision_ref: FirePdfTrainingCapture["decision_ref"],
+ ): FirePdfTrainingCapture => ({
+ eligible: decision_ref === "eligible",
+ policy_ref: "firecrawl-org-training-v1",
+ decision_ref,
+ });
+ const { teamFlags: flags, orgId } = meta.internalOptions;
+ if (meta.internalOptions.zeroDataRetention === true) return decision("zdr");
+ if (flags === undefined || typeof orgId !== "string")
+ return decision("org_unresolved");
+ if (flags?.trainingRestricted === true)
+ return decision("training_restricted");
+ return decision("eligible");
+}
+
+/** fire-pdf accepts a job's training decision only from an authenticated
+ * caller; without a key the job carries none and is never captured. */
+export function buildFirePdfJobTrainingCapture(
+ meta: Meta,
+): FirePdfTrainingCapture | undefined {
+ return config.FIRE_PDF_API_KEY
+ ? buildFirePdfTrainingCapture(meta)
+ : undefined;
+}
+
/** How the document reached us: an upload to the parse endpoint, or a scrape. */
export function firePdfRequestKind(meta: Meta): "scrape" | "parse" {
return meta.internalOptions.isParse === true ||
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index 540472b50..448d55973 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -3,6 +3,7 @@ import type { PDFMode } from "../../../../../controllers/v2/types";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
import {
+ buildFirePdfJobTrainingCapture,
buildFirePdfRequestMetadata,
firePdfTableFormat,
} from "./request-metadata";
@@ -161,6 +162,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
includeBlocks,
pageMarkers,
tableFormat: firePdfTableFormat(meta),
+ trainingCapture: buildFirePdfJobTrainingCapture(meta),
}),
};
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
index 82b0c67f7..37af82d48 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
@@ -1,5 +1,6 @@
import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
+import type { FirePdfTrainingCapture } from "./request-metadata";
import { config } from "../../../../../config";
import {
INLINE_JOB_DEADLINE_MARGIN_FRACTION,
@@ -159,6 +160,7 @@ export function buildFirePdfJobOptions(args: {
includeBlocks: boolean;
pageMarkers: boolean;
tableFormat: string | undefined;
+ trainingCapture: FirePdfTrainingCapture | undefined;
}): Record<string, unknown> {
return {
// fire-pdf accepts only a positive integer here. When the router has no
@@ -181,6 +183,7 @@ export function buildFirePdfJobOptions(args: {
// unknown option.
...(args.pageMarkers && { pageMarkers: true }),
...(args.tableFormat !== undefined && { tableFormat: args.tableFormat }),
+ ...(args.trainingCapture && { training_capture: args.trainingCapture }),
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index dd6e7015a..19839f72e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -14,6 +14,7 @@ import {
import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";
import {
buildFirePdfRequestMetadata,
+ buildFirePdfTrainingCapture,
firePdfTableFormat,
type FirePdfSourceKind,
} from "./fire-pdf/request-metadata";
@@ -154,6 +155,7 @@ export async function scrapePDFWithFirePDF(
pdf_sha256: pdfSha256,
source: "firecrawl",
zdr,
+ training_capture: buildFirePdfTrainingCapture(meta),
...deadlineFields,
},
logger,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 15d8678cc..0d220cd78 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -618,7 +618,9 @@ export type InternalOptions = {
saveScrapeResultToGCS?: boolean; // Passed along to fire-engine
bypassBilling?: boolean;
zeroDataRetention?: boolean;
- teamFlags?: TeamFlags;
+ /** null: the team has no flags. Absent: not snapshotted, so a check that
+ * must fail closed treats the org as unresolved. */
+ teamFlags?: TeamFlags | null;
/** Team's org, snapshotted from the request ACUC at acceptance (same
* pattern as teamFlags). Rides the job payload so org-scoped blocklist
* checks work without re-fetching the chunk. Required so a payload