Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(pdf): separate FirePDF async cohort for crawl/batch-originated scrapes (#4406)

Abimael Martelld3e7af2cf94d
diff --git a/.gitignore b/.gitignore
index 7d2396ed9..3c9c83018 100644
--- a/.gitignore
+++ b/.gitignore
@@ -62,3 +62,6 @@ _local/
apps/api/src/lib/scrape-interact/*.md
.pnpm-store
+
+# stray test runner logs
+vitest.out
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 22a7fe05f..3565d93b7 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -5,23 +5,28 @@ import { getMcpActionLogConfigErrors } from "./lib/mcp-action-log-config";
/* Codecs */
const delimitedList = (separator = ",") => {
return z.codec(z.string(), z.array(z.string()), {
- decode: (str) => (str ? str.split(separator).map((s) => s.trim()) : []),
- encode: (arr) => arr.join(separator),
+ decode: str => (str ? str.split(separator).map(s => s.trim()) : []),
+ encode: arr => arr.join(separator),
});
};
const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>
- z.preprocess((value) => (value === "" ? undefined : value), schema.optional());
+ z.preprocess(value => (value === "" ? undefined : value), schema.optional());
const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>
- z.preprocess((value) => (value === "" ? undefined : value), schema);
+ z.preprocess(value => (value === "" ? undefined : value), schema);
-const RESEARCH_PAPER_OPERATIONS = ["search", "inspect", "read", "similar"] as const;
+const RESEARCH_PAPER_OPERATIONS = [
+ "search",
+ "inspect",
+ "read",
+ "similar",
+] as const;
export type ResearchPaperOperation = (typeof RESEARCH_PAPER_OPERATIONS)[number];
const researchKeylessDisabled = z.preprocess(
- (value) => {
+ value => {
if (typeof value !== "string") return value;
const raw = value.trim().toLowerCase();
if (raw === "") return undefined;
@@ -31,10 +36,12 @@ const researchKeylessDisabled = z.preprocess(
}
return raw
.split(",")
- .map((operation) => operation.trim())
+ .map(operation => operation.trim())
.filter(Boolean);
},
- z.array(z.enum(RESEARCH_PAPER_OPERATIONS)).default([...RESEARCH_PAPER_OPERATIONS]),
+ z
+ .array(z.enum(RESEARCH_PAPER_OPERATIONS))
+ .default([...RESEARCH_PAPER_OPERATIONS]),
);
const containsLoneSurrogate = (value: string): boolean => {
@@ -67,11 +74,11 @@ const configSchema = z.object({
SUPPORT_AGENT_URL: z.string().url().optional(),
SUPPORT_AGENT_VERCEL_BYPASS_SECRET: z.string().optional(),
FIREBRAIN_TRACKS_URL: z.preprocess(
- (v) => (typeof v === "string" && v.trim() === "" ? undefined : v),
+ v => (typeof v === "string" && v.trim() === "" ? undefined : v),
z.string().url().optional(),
),
FIREBRAIN_TRACKS_API_KEY: z.preprocess(
- (v) => (typeof v === "string" && v.trim() === "" ? undefined : v),
+ v => (typeof v === "string" && v.trim() === "" ? undefined : v),
z.string().trim().optional(),
),
RESEARCH_PROXY_URL: z.string().url().optional(),
@@ -108,14 +115,21 @@ const configSchema = z.object({
// Google Web Risk. An unset key disables the provider (lookups then fail
// per the org's failurePolicy).
GOOGLE_WEB_RISK_API_KEY: z.string().optional(),
- GOOGLE_WEB_RISK_API_URL: z.string().url().default("https://webrisk.googleapis.com"),
+ GOOGLE_WEB_RISK_API_URL: z
+ .string()
+ .url()
+ .default("https://webrisk.googleapis.com"),
// Google Web Risk Update API sync tuning. ZDR: "normal" mode checks run
// against a locally synced hash-prefix database (threatLists:computeDiff)
// instead of sending URLs to Google, and verdicts are never persisted.
//
// Floor for how often threatLists:computeDiff may run per list. Google's
// recommendedNextDiff is respected when it is later than this floor.
- THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce.number().int().positive().default(60),
+ THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(60),
// A synced threat list older than this is treated as unavailable
// (provider-failure semantics → the org's failurePolicy decides).
THREAT_LIST_STALENESS_SECONDS: z.coerce
@@ -138,7 +152,9 @@ const configSchema = z.object({
// API Keys & Authentication
BULL_AUTH_KEY: z.string().optional(),
- S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),
+ S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY: emptyStringAsUndefined(
+ z.string().trim().min(1),
+ ),
OPENAI_API_KEY: z.string().optional(),
OPENAI_BASE_URL: z.string().optional(),
OPENROUTER_API_KEY: z.string().optional(),
@@ -151,7 +167,11 @@ const configSchema = z.object({
SEARCH_PREVIEW_TOKEN: z.string().optional(),
SEARCH_SERVICE_API_SECRET: z.string().optional(),
SEARCH_FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(120),
- SEARCH_FEEDBACK_DAILY_CAP_CREDITS: z.coerce.number().int().nonnegative().default(100),
+ SEARCH_FEEDBACK_DAILY_CAP_CREDITS: z.coerce
+ .number()
+ .int()
+ .nonnegative()
+ .default(100),
FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(120),
FEEDBACK_DAILY_CAP_CREDITS: z.coerce.number().int().nonnegative().default(50),
FEEDBACK_REFUND_ENABLED: z.stringbool().default(true),
@@ -191,11 +211,17 @@ const configSchema = z.object({
NUQ_RABBITMQ_URL: z.string().optional(),
FDB_CLUSTER_FILE: emptyStringAsUndefined(z.string()),
NUQ_BACKEND: emptyStringAsUndefined(z.enum(["pg", "fdb"])),
- NUQ_FDB_READY_SHARDS: emptyStringAsDefault(z.coerce.number().int().positive().default(2048)),
+ NUQ_FDB_READY_SHARDS: emptyStringAsDefault(
+ z.coerce.number().int().positive().default(2048),
+ ),
// 1 = strict (priority, FIFO) promotion order per team; raise for teams with
// extreme finish rates at the cost of approximate cross-shard ordering
- NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(z.coerce.number().int().positive().default(1)),
- NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(z.coerce.number().int().positive().default(16)),
+ NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(
+ z.coerce.number().int().positive().default(1),
+ ),
+ NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(
+ z.coerce.number().int().positive().default(16),
+ ),
// Google Cloud Storage
GCS_BUCKET_NAME: z.string().optional(),
@@ -305,6 +331,14 @@ const configSchema = z.object({
// Async /jobs rollout is a separate, server-controlled cohort inside
// traffic already selected for FirePDF. It is disabled by default.
FIRE_PDF_ASYNC_PERCENT: z.coerce.number().min(0).max(100).default(0),
+ // Separate cohort for crawl/batch-originated scrapes (any scrape
+ // carrying a crawlId): no caller waits on one specific document, so
+ // these can ramp onto the async lane ahead of interactive traffic.
+ FIRE_PDF_ASYNC_BULK_ORIGIN_PERCENT: z.coerce
+ .number()
+ .min(0)
+ .max(100)
+ .default(0),
FIRE_PDF_ASYNC_FORCE_TEAM_IDS: z.string().optional(),
FIRE_PDF_ASYNC_DISABLE_TEAM_IDS: z.string().optional(),
FIRE_PDF_ASYNC_ALLOW_REQUEST_OVERRIDE: z.stringbool().default(false),
@@ -423,10 +457,10 @@ const configSchema = z.object({
EXTRACT_V3_BETA_URL: z.string().optional(),
AGENT_INTEROP_SECRET: z
.string()
- .refine((value) => value.trim().length > 0, {
+ .refine(value => value.trim().length > 0, {
error: "AGENT_INTEROP_SECRET must not be blank",
})
- .refine((value) => !containsLoneSurrogate(value), {
+ .refine(value => !containsLoneSurrogate(value), {
error: "AGENT_INTEROP_SECRET must not contain lone surrogates",
})
.optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts
index 9a24b0627..20e74b52f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts
@@ -80,6 +80,63 @@ describe("FirePDF async routing", () => {
reason: "percentage",
});
});
+
+ it("routes crawl/batch children on their own cohort", () => {
+ expect(
+ decideFirePdfAsyncRoute({
+ ...baseInput,
+ bulkOrigin: true,
+ bulkOriginPercentage: 100,
+ }),
+ ).toEqual({ enabled: true, reason: "bulk_origin" });
+ // Traffic-neutral by default, like every other cohort.
+ expect(decideFirePdfAsyncRoute({ ...baseInput, bulkOrigin: true })).toEqual(
+ { enabled: false, reason: "percentage_disabled" },
+ );
+ // Never applies to non-bulk scrapes.
+ expect(
+ decideFirePdfAsyncRoute({ ...baseInput, bulkOriginPercentage: 100 }),
+ ).toEqual({ enabled: false, reason: "percentage_disabled" });
+ });
+
+ it("keeps the hard exclusions ahead of the bulk cohort", () => {
+ const bulk = {
+ ...baseInput,
+ bulkOrigin: true,
+ bulkOriginPercentage: 100,
+ };
+ expect(
+ decideFirePdfAsyncRoute({ ...bulk, zeroDataRetention: true }),
+ ).toEqual({ enabled: false, reason: "zdr" });
+ expect(
+ decideFirePdfAsyncRoute({
+ ...bulk,
+ remainingMs: FIRE_PDF_ASYNC_MIN_REMAINING_MS - 1,
+ }),
+ ).toEqual({ enabled: false, reason: "deadline_too_close" });
+ expect(
+ decideFirePdfAsyncRoute({ ...bulk, disableTeamIds: "team-1" }),
+ ).toEqual({ enabled: false, reason: "team_disabled" });
+ });
+
+ it("cohorts bulk and general percentages independently", () => {
+ // The bulk cohort hashes a prefixed key, so a scrape's position in
+ // one cohort says nothing about its position in the other.
+ const scrapeId = "cohort-independence-probe";
+ expect(deterministicPercentage(`bulk-origin:${scrapeId}`)).not.toBe(
+ deterministicPercentage(scrapeId),
+ );
+ // A bulk scrape outside its cohort still falls through to the
+ // general percentage.
+ expect(
+ decideFirePdfAsyncRoute({
+ ...baseInput,
+ bulkOrigin: true,
+ bulkOriginPercentage: 0,
+ percentage: 100,
+ }),
+ ).toEqual({ enabled: true, reason: "percentage" });
+ });
});
describe("FirePDF async transport helpers", () => {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts
index b4ffd5342..958b96e04 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts
@@ -9,6 +9,7 @@ type FirePdfAsyncRouteReason =
| "team_disabled"
| "team_forced"
| "request_override"
+ | "bulk_origin"
| "percentage"
| "percentage_disabled"
| "outside_percentage";
@@ -28,6 +29,11 @@ type FirePdfAsyncRouteInput = {
forceTeamIds?: string;
disableTeamIds?: string;
allowRequestOverride: boolean;
+ /** Scrape is a child of a crawl or batch scrape (carries a crawlId).
+ * No caller is blocked on this specific document, so it can ride the
+ * async lane under its own, separately ramped percentage. */
+ bulkOrigin?: boolean;
+ bulkOriginPercentage?: number;
};
function parseTeamIds(value: string | undefined): Set<string> {
@@ -69,6 +75,19 @@ export function decideFirePdfAsyncRoute(
return { enabled: true, reason: "request_override" };
}
+ // Crawl/batch children ramp on their own cohort. The hash is keyed
+ // separately from the general cohort so the two percentages stay
+ // independent; a bulk scrape outside its cohort still falls through
+ // to the general percentage below.
+ if (
+ input.bulkOrigin &&
+ (input.bulkOriginPercentage ?? 0) > 0 &&
+ deterministicPercentage(`bulk-origin:${input.scrapeId}`) <
+ (input.bulkOriginPercentage ?? 0)
+ ) {
+ return { enabled: true, reason: "bulk_origin" };
+ }
+
if (input.percentage <= 0) {
return { enabled: false, reason: "percentage_disabled" };
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index bc67579ea..9c6536dbf 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -452,6 +452,8 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
forceTeamIds: config.FIRE_PDF_ASYNC_FORCE_TEAM_IDS,
disableTeamIds: config.FIRE_PDF_ASYNC_DISABLE_TEAM_IDS,
allowRequestOverride: config.FIRE_PDF_ASYNC_ALLOW_REQUEST_OVERRIDE,
+ bulkOrigin: Boolean(meta.internalOptions.crawlId),
+ bulkOriginPercentage: config.FIRE_PDF_ASYNC_BULK_ORIGIN_PERCENT,
});
const useAsync = asyncDecision.enabled;
if (useAsync) {
@@ -460,8 +462,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
event: "fire_pdf_async_routed",
reason: asyncDecision.reason,
percentage: config.FIRE_PDF_ASYNC_PERCENT,
+ bulk_origin_percentage: config.FIRE_PDF_ASYNC_BULK_ORIGIN_PERCENT,
scrape_id: meta.id,
team_id: meta.internalOptions.teamId,
+ crawl_id: meta.internalOptions.crawlId,
});
}
try {