Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): send FirePDF a per-org training decision (#5395)
Gergő Móriczc351eb6dac46
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 27f103c06..b637819a4 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -353,7 +353,7 @@ export async function batchScrapeController(agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,// Safe Mode resolves per-URL at the scrapeURL backstop from these flags.- teamFlags: req.acuc?.flags ?? undefined,+ teamFlags: req.acuc?.flags ?? null,safeModeBypassed: safeMode.bypassed === true,}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 5f709a9d8..3c042e5d7 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -273,7 +273,7 @@ export async function crawlController(agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,// Safe Mode resolves per-URL at the scrapeURL backstop from these flags.- teamFlags: req.acuc?.flags ?? undefined,+ teamFlags: req.acuc?.flags ?? null,safeModeBypassed: safeMode.bypassed === true,}, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex e259a5268..2a20a0032 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1406,6 +1406,9 @@ export type TeamFlags = {// The org's DPA (or partner amendment) restricts how its data may be// handled. Informational only: the API does not change behavior on it.dpaRestricted?: boolean;+ // The org's contract bars training on its data: FirePDF never captures its+ // documents (buildFirePdfTrainingCapture).+ trainingRestricted?: boolean;organizationDataSourceAccess?: Record<string,{diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex d6d4bfc55..ea304cad1 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -442,7 +442,7 @@ export async function batchScrapeController(threatProtection: threatProtection.policy ?? undefined,// Safe Mode rides the batch payload so each URL resolves it per-URL// at the scrapeURL backstop.- teamFlags: req.acuc?.flags ?? undefined,+ teamFlags: req.acuc?.flags ?? null,safeModeBypassed: safeMode.bypassed === true,}, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 0f24298c6..1438ae7c6 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -343,7 +343,7 @@ export async function crawlController(threatProtection: threatProtection.policy ?? undefined,// Safe Mode rides the crawl payload so every child scrape resolves it// per-URL at the scrapeURL backstop (allowlist applies per child).- teamFlags: req.acuc?.flags ?? undefined,+ teamFlags: req.acuc?.flags ?? null,safeModeBypassed: safeMode.bypassed === true,},team_id: req.auth.team_id,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex a96ed5e2c..ef554b1b1 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -2173,6 +2173,9 @@ export type TeamFlags = {// The org's DPA (or partner amendment) restricts how its data may be// handled. Informational only: the API does not change behavior on it.dpaRestricted?: boolean;+ // The org's contract bars training on its data: FirePDF never captures its+ // documents (buildFirePdfTrainingCapture).+ trainingRestricted?: boolean;organizationDataSourceAccess?: Record<string,{diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.tsindex 322cc52c1..e7eb4fcb4 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts@@ -469,6 +469,11 @@ describe("FirePDF table format", () => {);it("puts the format in async job options only when set", () => {+ const trainingCapture = {+ eligible: true,+ policy_ref: "firecrawl-org-training-v1",+ decision_ref: "eligible",+ } as const;const base = {maxPages: undefined,pagesProcessed: 2,@@ -476,12 +481,25 @@ describe("FirePDF table format", () => {includePageMarkdown: false,includeBlocks: false,pageMarkers: false,+ trainingCapture,};expect(buildFirePdfJobOptions({ ...base, tableFormat: "dynamic" })).toEqual(- { pages_estimate: 2, mode: "auto", tableFormat: "dynamic" },+ {+ pages_estimate: 2,+ mode: "auto",+ tableFormat: "dynamic",+ training_capture: trainingCapture,+ },);expect(buildFirePdfJobOptions({ ...base, tableFormat: undefined })).toEqual(- { pages_estimate: 2, mode: "auto" },+ { pages_estimate: 2, mode: "auto", training_capture: trainingCapture },);+ expect(+ buildFirePdfJobOptions({+ ...base,+ tableFormat: undefined,+ trainingCapture: undefined,+ }),+ ).toEqual({ pages_estimate: 2, mode: "auto" });});});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.tsindex 39b13a047..8dda6556b 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts@@ -10,7 +10,10 @@ import {import { cacheKeyShape, tryGetCached } from "./cache";import { lookupAdoptableFirePdfJob } from "./lookup";import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "./utils";-import { firePdfTableFormat } from "./request-metadata";+import {+ buildFirePdfJobTrainingCapture,+ firePdfTableFormat,+} from "./request-metadata";type FirePdfByReferenceAttemptArgs = {meta: Meta;@@ -147,6 +150,7 @@ export async function runFirePdfByReferenceAttempt(includeBlocks,pageMarkers,tableFormat: firePdfTableFormat(meta),+ trainingCapture: buildFirePdfJobTrainingCapture(meta),}),);meta.abort.throwIfAborted();diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.tsindex ea46723f9..b2bff279c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts@@ -6,6 +6,44 @@ import { hasCustomRequestContext } from "../../../lib/request-context";/** What we hand fire-pdf: a PDF, or a raster image wrapped as a one-page document. */export type FirePdfSourceKind = "pdf" | "image";+/** FirePDF's training decision; its capture lanes run only on an eligible one. */+export type FirePdfTrainingCapture = {+ eligible: boolean;+ policy_ref: "firecrawl-org-training-v1";+ decision_ref: "zdr" | "org_unresolved" | "training_restricted" | "eligible";+};++/** Fail closed: eligible only outside ZDR, for an org whose id and flags were+ * snapshotted at acceptance and whose flags carry no training restriction. */+export function buildFirePdfTrainingCapture(+ meta: Meta,+): FirePdfTrainingCapture {+ const decision = (+ decision_ref: FirePdfTrainingCapture["decision_ref"],+ ): FirePdfTrainingCapture => ({+ eligible: decision_ref === "eligible",+ policy_ref: "firecrawl-org-training-v1",+ decision_ref,+ });+ const { teamFlags: flags, orgId } = meta.internalOptions;+ if (meta.internalOptions.zeroDataRetention === true) return decision("zdr");+ if (flags === undefined || typeof orgId !== "string")+ return decision("org_unresolved");+ if (flags?.trainingRestricted === true)+ return decision("training_restricted");+ return decision("eligible");+}++/** fire-pdf accepts a job's training decision only from an authenticated+ * caller; without a key the job carries none and is never captured. */+export function buildFirePdfJobTrainingCapture(+ meta: Meta,+): FirePdfTrainingCapture | undefined {+ return config.FIRE_PDF_API_KEY+ ? buildFirePdfTrainingCapture(meta)+ : undefined;+}+/** How the document reached us: an upload to the parse endpoint, or a scrape. */export function firePdfRequestKind(meta: Meta): "scrape" | "parse" {return meta.internalOptions.isParse === true ||diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex 540472b50..448d55973 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -3,6 +3,7 @@ import type { PDFMode } from "../../../../../controllers/v2/types";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";import {+ buildFirePdfJobTrainingCapture,buildFirePdfRequestMetadata,firePdfTableFormat,} from "./request-metadata";@@ -161,6 +162,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {includeBlocks,pageMarkers,tableFormat: firePdfTableFormat(meta),+ trainingCapture: buildFirePdfJobTrainingCapture(meta),}),};diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.tsindex 82b0c67f7..37af82d48 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts@@ -1,5 +1,6 @@import type { Meta } from "../../..";import type { PDFMode } from "../../../../../controllers/v2/types";+import type { FirePdfTrainingCapture } from "./request-metadata";import { config } from "../../../../../config";import {INLINE_JOB_DEADLINE_MARGIN_FRACTION,@@ -159,6 +160,7 @@ export function buildFirePdfJobOptions(args: {includeBlocks: boolean;pageMarkers: boolean;tableFormat: string | undefined;+ trainingCapture: FirePdfTrainingCapture | undefined;}): Record<string, unknown> {return {// fire-pdf accepts only a positive integer here. When the router has no@@ -181,6 +183,7 @@ export function buildFirePdfJobOptions(args: {// unknown option....(args.pageMarkers && { pageMarkers: true }),...(args.tableFormat !== undefined && { tableFormat: args.tableFormat }),+ ...(args.trainingCapture && { training_capture: args.trainingCapture }),};}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex dd6e7015a..19839f72e 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -14,6 +14,7 @@ import {import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";import {buildFirePdfRequestMetadata,+ buildFirePdfTrainingCapture,firePdfTableFormat,type FirePdfSourceKind,} from "./fire-pdf/request-metadata";@@ -154,6 +155,7 @@ export async function scrapePDFWithFirePDF(pdf_sha256: pdfSha256,source: "firecrawl",zdr,+ training_capture: buildFirePdfTrainingCapture(meta),...deadlineFields,},logger,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 15d8678cc..0d220cd78 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -618,7 +618,9 @@ export type InternalOptions = {saveScrapeResultToGCS?: boolean; // Passed along to fire-enginebypassBilling?: boolean;zeroDataRetention?: boolean;- teamFlags?: TeamFlags;+ /** null: the team has no flags. Absent: not snapshotted, so a check that+ * must fail closed treats the org as unresolved. */+ teamFlags?: TeamFlags | null;/** Team's org, snapshotted from the request ACUC at acceptance (same* pattern as teamFlags). Rides the job payload so org-scoped blocklist* checks work without re-fetching the chunk. Required so a payload