Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf): separate FirePDF async cohort for crawl/batch-originated scrapes (#4406)
Abimael Martelld3e7af2cf94d
diff --git a/.gitignore b/.gitignoreindex 7d2396ed9..3c9c83018 100644--- a/.gitignore+++ b/.gitignore@@ -62,3 +62,6 @@ _local/apps/api/src/lib/scrape-interact/*.md.pnpm-store++# stray test runner logs+vitest.outdiff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 22a7fe05f..3565d93b7 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -5,23 +5,28 @@ import { getMcpActionLogConfigErrors } from "./lib/mcp-action-log-config";/* Codecs */const delimitedList = (separator = ",") => {return z.codec(z.string(), z.array(z.string()), {- decode: (str) => (str ? str.split(separator).map((s) => s.trim()) : []),- encode: (arr) => arr.join(separator),+ decode: str => (str ? str.split(separator).map(s => s.trim()) : []),+ encode: arr => arr.join(separator),});};const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>- z.preprocess((value) => (value === "" ? undefined : value), schema.optional());+ z.preprocess(value => (value === "" ? undefined : value), schema.optional());const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>- z.preprocess((value) => (value === "" ? undefined : value), schema);+ z.preprocess(value => (value === "" ? undefined : value), schema);-const RESEARCH_PAPER_OPERATIONS = ["search", "inspect", "read", "similar"] as const;+const RESEARCH_PAPER_OPERATIONS = [+ "search",+ "inspect",+ "read",+ "similar",+] as const;export type ResearchPaperOperation = (typeof RESEARCH_PAPER_OPERATIONS)[number];const researchKeylessDisabled = z.preprocess(- (value) => {+ value => {if (typeof value !== "string") return value;const raw = value.trim().toLowerCase();if (raw === "") return undefined;@@ -31,10 +36,12 @@ const researchKeylessDisabled = z.preprocess(}return raw.split(",")- .map((operation) => operation.trim())+ .map(operation => operation.trim()).filter(Boolean);},- z.array(z.enum(RESEARCH_PAPER_OPERATIONS)).default([...RESEARCH_PAPER_OPERATIONS]),+ z+ .array(z.enum(RESEARCH_PAPER_OPERATIONS))+ .default([...RESEARCH_PAPER_OPERATIONS]),);const containsLoneSurrogate = (value: string): boolean => {@@ -67,11 +74,11 @@ const configSchema = z.object({SUPPORT_AGENT_URL: z.string().url().optional(),SUPPORT_AGENT_VERCEL_BYPASS_SECRET: z.string().optional(),FIREBRAIN_TRACKS_URL: z.preprocess(- (v) => (typeof v === "string" && v.trim() === "" ? undefined : v),+ v => (typeof v === "string" && v.trim() === "" ? undefined : v),z.string().url().optional(),),FIREBRAIN_TRACKS_API_KEY: z.preprocess(- (v) => (typeof v === "string" && v.trim() === "" ? undefined : v),+ v => (typeof v === "string" && v.trim() === "" ? undefined : v),z.string().trim().optional(),),RESEARCH_PROXY_URL: z.string().url().optional(),@@ -108,14 +115,21 @@ const configSchema = z.object({// Google Web Risk. An unset key disables the provider (lookups then fail// per the org's failurePolicy).GOOGLE_WEB_RISK_API_KEY: z.string().optional(),- GOOGLE_WEB_RISK_API_URL: z.string().url().default("https://webrisk.googleapis.com"),+ GOOGLE_WEB_RISK_API_URL: z+ .string()+ .url()+ .default("https://webrisk.googleapis.com"),// Google Web Risk Update API sync tuning. ZDR: "normal" mode checks run// against a locally synced hash-prefix database (threatLists:computeDiff)// instead of sending URLs to Google, and verdicts are never persisted.//// Floor for how often threatLists:computeDiff may run per list. Google's// recommendedNextDiff is respected when it is later than this floor.- THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce.number().int().positive().default(60),+ THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce+ .number()+ .int()+ .positive()+ .default(60),// A synced threat list older than this is treated as unavailable// (provider-failure semantics → the org's failurePolicy decides).THREAT_LIST_STALENESS_SECONDS: z.coerce@@ -138,7 +152,9 @@ const configSchema = z.object({// API Keys & AuthenticationBULL_AUTH_KEY: z.string().optional(),- S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),+ S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY: emptyStringAsUndefined(+ z.string().trim().min(1),+ ),OPENAI_API_KEY: z.string().optional(),OPENAI_BASE_URL: z.string().optional(),OPENROUTER_API_KEY: z.string().optional(),@@ -151,7 +167,11 @@ const configSchema = z.object({SEARCH_PREVIEW_TOKEN: z.string().optional(),SEARCH_SERVICE_API_SECRET: z.string().optional(),SEARCH_FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(120),- SEARCH_FEEDBACK_DAILY_CAP_CREDITS: z.coerce.number().int().nonnegative().default(100),+ SEARCH_FEEDBACK_DAILY_CAP_CREDITS: z.coerce+ .number()+ .int()+ .nonnegative()+ .default(100),FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(120),FEEDBACK_DAILY_CAP_CREDITS: z.coerce.number().int().nonnegative().default(50),FEEDBACK_REFUND_ENABLED: z.stringbool().default(true),@@ -191,11 +211,17 @@ const configSchema = z.object({NUQ_RABBITMQ_URL: z.string().optional(),FDB_CLUSTER_FILE: emptyStringAsUndefined(z.string()),NUQ_BACKEND: emptyStringAsUndefined(z.enum(["pg", "fdb"])),- NUQ_FDB_READY_SHARDS: emptyStringAsDefault(z.coerce.number().int().positive().default(2048)),+ NUQ_FDB_READY_SHARDS: emptyStringAsDefault(+ z.coerce.number().int().positive().default(2048),+ ),// 1 = strict (priority, FIFO) promotion order per team; raise for teams with// extreme finish rates at the cost of approximate cross-shard ordering- NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(z.coerce.number().int().positive().default(1)),- NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(z.coerce.number().int().positive().default(16)),+ NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(+ z.coerce.number().int().positive().default(1),+ ),+ NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(+ z.coerce.number().int().positive().default(16),+ ),// Google Cloud StorageGCS_BUCKET_NAME: z.string().optional(),@@ -305,6 +331,14 @@ const configSchema = z.object({// Async /jobs rollout is a separate, server-controlled cohort inside// traffic already selected for FirePDF. It is disabled by default.FIRE_PDF_ASYNC_PERCENT: z.coerce.number().min(0).max(100).default(0),+ // Separate cohort for crawl/batch-originated scrapes (any scrape+ // carrying a crawlId): no caller waits on one specific document, so+ // these can ramp onto the async lane ahead of interactive traffic.+ FIRE_PDF_ASYNC_BULK_ORIGIN_PERCENT: z.coerce+ .number()+ .min(0)+ .max(100)+ .default(0),FIRE_PDF_ASYNC_FORCE_TEAM_IDS: z.string().optional(),FIRE_PDF_ASYNC_DISABLE_TEAM_IDS: z.string().optional(),FIRE_PDF_ASYNC_ALLOW_REQUEST_OVERRIDE: z.stringbool().default(false),@@ -423,10 +457,10 @@ const configSchema = z.object({EXTRACT_V3_BETA_URL: z.string().optional(),AGENT_INTEROP_SECRET: z.string()- .refine((value) => value.trim().length > 0, {+ .refine(value => value.trim().length > 0, {error: "AGENT_INTEROP_SECRET must not be blank",})- .refine((value) => !containsLoneSurrogate(value), {+ .refine(value => !containsLoneSurrogate(value), {error: "AGENT_INTEROP_SECRET must not contain lone surrogates",}).optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.tsindex 9a24b0627..20e74b52f 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts@@ -80,6 +80,63 @@ describe("FirePDF async routing", () => {reason: "percentage",});});++ it("routes crawl/batch children on their own cohort", () => {+ expect(+ decideFirePdfAsyncRoute({+ ...baseInput,+ bulkOrigin: true,+ bulkOriginPercentage: 100,+ }),+ ).toEqual({ enabled: true, reason: "bulk_origin" });+ // Traffic-neutral by default, like every other cohort.+ expect(decideFirePdfAsyncRoute({ ...baseInput, bulkOrigin: true })).toEqual(+ { enabled: false, reason: "percentage_disabled" },+ );+ // Never applies to non-bulk scrapes.+ expect(+ decideFirePdfAsyncRoute({ ...baseInput, bulkOriginPercentage: 100 }),+ ).toEqual({ enabled: false, reason: "percentage_disabled" });+ });++ it("keeps the hard exclusions ahead of the bulk cohort", () => {+ const bulk = {+ ...baseInput,+ bulkOrigin: true,+ bulkOriginPercentage: 100,+ };+ expect(+ decideFirePdfAsyncRoute({ ...bulk, zeroDataRetention: true }),+ ).toEqual({ enabled: false, reason: "zdr" });+ expect(+ decideFirePdfAsyncRoute({+ ...bulk,+ remainingMs: FIRE_PDF_ASYNC_MIN_REMAINING_MS - 1,+ }),+ ).toEqual({ enabled: false, reason: "deadline_too_close" });+ expect(+ decideFirePdfAsyncRoute({ ...bulk, disableTeamIds: "team-1" }),+ ).toEqual({ enabled: false, reason: "team_disabled" });+ });++ it("cohorts bulk and general percentages independently", () => {+ // The bulk cohort hashes a prefixed key, so a scrape's position in+ // one cohort says nothing about its position in the other.+ const scrapeId = "cohort-independence-probe";+ expect(deterministicPercentage(`bulk-origin:${scrapeId}`)).not.toBe(+ deterministicPercentage(scrapeId),+ );+ // A bulk scrape outside its cohort still falls through to the+ // general percentage.+ expect(+ decideFirePdfAsyncRoute({+ ...baseInput,+ bulkOrigin: true,+ bulkOriginPercentage: 0,+ percentage: 100,+ }),+ ).toEqual({ enabled: true, reason: "percentage" });+ });});describe("FirePDF async transport helpers", () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.tsindex b4ffd5342..958b96e04 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts@@ -9,6 +9,7 @@ type FirePdfAsyncRouteReason =| "team_disabled"| "team_forced"| "request_override"+ | "bulk_origin"| "percentage"| "percentage_disabled"| "outside_percentage";@@ -28,6 +29,11 @@ type FirePdfAsyncRouteInput = {forceTeamIds?: string;disableTeamIds?: string;allowRequestOverride: boolean;+ /** Scrape is a child of a crawl or batch scrape (carries a crawlId).+ * No caller is blocked on this specific document, so it can ride the+ * async lane under its own, separately ramped percentage. */+ bulkOrigin?: boolean;+ bulkOriginPercentage?: number;};function parseTeamIds(value: string | undefined): Set<string> {@@ -69,6 +75,19 @@ export function decideFirePdfAsyncRoute(return { enabled: true, reason: "request_override" };}+ // Crawl/batch children ramp on their own cohort. The hash is keyed+ // separately from the general cohort so the two percentages stay+ // independent; a bulk scrape outside its cohort still falls through+ // to the general percentage below.+ if (+ input.bulkOrigin &&+ (input.bulkOriginPercentage ?? 0) > 0 &&+ deterministicPercentage(`bulk-origin:${input.scrapeId}`) <+ (input.bulkOriginPercentage ?? 0)+ ) {+ return { enabled: true, reason: "bulk_origin" };+ }+if (input.percentage <= 0) {return { enabled: false, reason: "percentage_disabled" };}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex bc67579ea..9c6536dbf 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -452,6 +452,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {forceTeamIds: config.FIRE_PDF_ASYNC_FORCE_TEAM_IDS,disableTeamIds: config.FIRE_PDF_ASYNC_DISABLE_TEAM_IDS,allowRequestOverride: config.FIRE_PDF_ASYNC_ALLOW_REQUEST_OVERRIDE,+ bulkOrigin: Boolean(meta.internalOptions.crawlId),+ bulkOriginPercentage: config.FIRE_PDF_ASYNC_BULK_ORIGIN_PERCENT,});const useAsync = asyncDecision.enabled;if (useAsync) {@@ -460,8 +462,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {event: "fire_pdf_async_routed",reason: asyncDecision.reason,percentage: config.FIRE_PDF_ASYNC_PERCENT,+ bulk_origin_percentage: config.FIRE_PDF_ASYNC_BULK_ORIGIN_PERCENT,scrape_id: meta.id,team_id: meta.internalOptions.teamId,+ crawl_id: meta.internalOptions.crawlId,});}try {