Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): optional extraction provider for JSON mode with cohort rollout, shadow mode and per-request fallback (#5173)

Abimael Martell5b539721c72a
diff --git a/apps/api/.env.example b/apps/api/.env.example
index d9265d931..e848b75a9 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -81,6 +81,20 @@ TYPESAFE_API_KEY=
# BRANDING_JEV_ESCALATE_BELOW=
# Share of LLM-answered branding requests that also ask Jev in the background to compare answers (response unchanged)
# BRANDING_JEV_SHADOW_PERCENT=0
+# optional OpenAI-compatible provider for JSON extraction; routed by team cohort
+# at ROLLOUT_PERCENT, mirrored at SHADOW_PERCENT, with per-request fallback
+EXTRACTION_PROVIDER_BASE_URL=
+EXTRACTION_PROVIDER_API_KEY=
+EXTRACTION_PROVIDER_MODEL=extraction-v1
+EXTRACTION_PROVIDER_ROLLOUT_PERCENT=0
+EXTRACTION_PROVIDER_SHADOW_PERCENT=0
+EXTRACTION_PROVIDER_TEAM_IDS=
+# EXTRACTION_PROVIDER_REQUEST_OPT_IN=false
+# inputs above this many bytes stay on the default model
+# EXTRACTION_PROVIDER_MAX_INPUT_BYTES=400000
+# amortized price of the provider model, per million tokens, for cost tracking
+# EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK=0.07
+# EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK=0.2
BULL_AUTH_KEY=@
# set if you have a llamaparse key you'd like to use to parse pdfs
LLAMAPARSE_API_KEY=
diff --git a/apps/api/src/__tests__/snips/v2/extraction-provider.test.ts b/apps/api/src/__tests__/snips/v2/extraction-provider.test.ts
new file mode 100644
index 000000000..7d278f5ad
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/extraction-provider.test.ts
@@ -0,0 +1,84 @@
+import {
+ scrape,
+ scrapeWithFailure,
+ scrapeTimeout,
+ idmux,
+ Identity,
+} from "./lib";
+
+// Runs only where the API under test is guaranteed to route to the provider:
+// a configured base URL and either a 100% rollout or the per-request opt-in.
+const configured =
+ !!process.env.EXTRACTION_PROVIDER_BASE_URL &&
+ (process.env.EXTRACTION_PROVIDER_ROLLOUT_PERCENT === "100" ||
+ ["true", "1"].includes(
+ (process.env.EXTRACTION_PROVIDER_REQUEST_OPT_IN ?? "").toLowerCase(),
+ ));
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "extraction-provider",
+ concurrency: 10,
+ credits: 1_000_000,
+ });
+}, 10000);
+
+(configured ? describe : describe.skip)(
+ "JSON extraction via the extraction provider",
+ () => {
+ it.concurrent(
+ "returns a schema-conforming object",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://firecrawl.dev",
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: {
+ company_name: { type: "string" },
+ tagline: { type: "string" },
+ },
+ required: ["company_name", "tagline"],
+ },
+ },
+ ],
+ },
+ identity,
+ );
+ expect(response.json).toBeDefined();
+ expect(typeof response.json.company_name).toBe("string");
+ expect(typeof response.json.tagline).toBe("string");
+ expect(response.json.company_name.toLowerCase()).toContain("firecrawl");
+ },
+ scrapeTimeout,
+ );
+ },
+);
+
+describe("JSON format model option", () => {
+ it.concurrent(
+ "rejects a model that is not available",
+ async () => {
+ const response = await scrapeWithFailure(
+ {
+ url: "https://firecrawl.dev",
+ formats: [
+ {
+ type: "json",
+ model: "not-a-configured-extraction-model",
+ schema: { type: "object", properties: { x: { type: "string" } } },
+ },
+ ],
+ },
+ identity,
+ );
+ expect(response.error).toContain("Unsupported extraction model");
+ },
+ scrapeTimeout,
+ );
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index fbb683faa..bf91f975d 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -16,6 +16,18 @@ const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>
const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>
z.preprocess(value => (value === "" ? undefined : value), schema);
+// Trim strings and treat blank ones as unset (optional or default schemas).
+const blankStringAsUnset = <T extends z.ZodTypeAny>(schema: T) =>
+ z.preprocess(
+ value =>
+ typeof value === "string"
+ ? value.trim() === ""
+ ? undefined
+ : value.trim()
+ : value,
+ schema,
+ );
+
const RESEARCH_PAPER_OPERATIONS = [
"search",
"inspect",
@@ -591,6 +603,44 @@ const configSchema = z.object({
// AI/ML
MODEL_NAME: z.string().optional(),
+ // Optional OpenAI-compatible provider for JSON extraction. Traffic is routed
+ // by team cohort (stable hash) at ROLLOUT_PERCENT, mirrored without using the
+ // result at SHADOW_PERCENT, and always routed for TEAM_IDS. Any failure falls
+ // back to the default model for that request.
+ EXTRACTION_PROVIDER_BASE_URL: blankStringAsUnset(z.string().url().optional()),
+ EXTRACTION_PROVIDER_API_KEY: blankStringAsUnset(z.string().optional()),
+ EXTRACTION_PROVIDER_MODEL: blankStringAsUnset(
+ z.string().min(1).default("extraction-v1"),
+ ),
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: z.coerce
+ .number()
+ .int()
+ .min(0)
+ .max(100)
+ .default(0),
+ EXTRACTION_PROVIDER_SHADOW_PERCENT: z.coerce
+ .number()
+ .int()
+ .min(0)
+ .max(100)
+ .default(0),
+ EXTRACTION_PROVIDER_TEAM_IDS: z.string().optional(),
+ // Let any team pick the provider per request (json format `model`); off =
+ // only teams in TEAM_IDS may.
+ EXTRACTION_PROVIDER_REQUEST_OPT_IN: z.stringbool().optional(),
+ EXTRACTION_PROVIDER_MAX_INPUT_BYTES: z.coerce
+ .number()
+ .int()
+ .nonnegative()
+ .default(400_000),
+ EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK: z.coerce
+ .number()
+ .nonnegative()
+ .default(0.07),
+ EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK: z.coerce
+ .number()
+ .nonnegative()
+ .default(0.2),
MODEL_EMBEDDING_NAME: z.string().optional(),
OLLAMA_BASE_URL: z.string().optional(),
VERTEX_CREDENTIALS: z.string().optional(),
@@ -721,6 +771,18 @@ const configSchema = z.object({
});
const validatedConfigSchema = configSchema.superRefine((value, context) => {
+ if (
+ value.EXTRACTION_PROVIDER_ROLLOUT_PERCENT +
+ value.EXTRACTION_PROVIDER_SHADOW_PERCENT >
+ 100
+ ) {
+ context.addIssue({
+ code: "custom",
+ path: ["EXTRACTION_PROVIDER_SHADOW_PERCENT"],
+ message:
+ "EXTRACTION_PROVIDER_ROLLOUT_PERCENT + EXTRACTION_PROVIDER_SHADOW_PERCENT must not exceed 100",
+ });
+ }
for (const error of getMcpActionLogConfigErrors(value)) {
context.addIssue({
code: "custom",
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 752950416..2c9e875e0 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -3,6 +3,7 @@ import type { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";
import { Request, Response } from "express";
import { hasCategory } from "../../lib/search-query-builder";
import { config } from "../../config";
+import { isRequestedModelSupported } from "../../lib/extraction-provider";
import { z } from "zod";
import { browserProfileNameSchema } from "../../lib/browser-profiles";
import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
@@ -365,6 +366,14 @@ const jsonFormatWithOptions = z.strictObject({
})
.superRefine(addStrictSchemaIssue),
prompt: z.string().max(10000).optional(),
+ // Optional: name a configured extraction model for this request.
+ model: z
+ .string()
+ .max(100)
+ .optional()
+ .refine(val => val === undefined || isRequestedModelSupported(val), {
+ message: "Unsupported extraction model.",
+ }),
// Deprecated: lifted into the top-level checkPromptInjection at parse time.
checkPromptInjection: z.boolean().optional(),
});
diff --git a/apps/api/src/lib/extraction-provider.test.ts b/apps/api/src/lib/extraction-provider.test.ts
new file mode 100644
index 000000000..1a4dfd473
--- /dev/null
+++ b/apps/api/src/lib/extraction-provider.test.ts
@@ -0,0 +1,200 @@
+import { config } from "../config";
+import {
+ cohortBucket,
+ extractionProviderCost,
+ isRequestedModelSupported,
+ selectExtractionRoute,
+} from "./extraction-provider";
+
+const base = { teamId: "team-a", zeroDataRetention: false, inputBytes: 10_000 };
+
+function withConfig<T>(overrides: Partial<typeof config>, fn: () => T): T {
+ const saved: Partial<typeof config> = {};
+ for (const k of Object.keys(overrides) as (keyof typeof config)[]) {
+ (saved as any)[k] = config[k];
+ (config as any)[k] = overrides[k];
+ }
+ try {
+ return fn();
+ } finally {
+ Object.assign(config, saved);
+ }
+}
+
+describe("extraction provider routing", () => {
+ it("is off when no provider is configured", () => {
+ withConfig(
+ {
+ EXTRACTION_PROVIDER_BASE_URL: undefined,
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,
+ },
+ () => expect(selectExtractionRoute(base)).toBe("default"),
+ );
+ });
+
+ it("buckets teams stably and evenly enough to roll out by percentage", () => {
+ const a = cohortBucket("team-a");
+ expect(cohortBucket("team-a")).toBe(a);
+ expect(a).toBeGreaterThanOrEqual(0);
+ expect(a).toBeLessThan(100);
+ const buckets = new Set(
+ Array.from({ length: 2000 }, (_, i) => cohortBucket(`team-${i}`)),
+ );
+ expect(buckets.size).toBe(100);
+ });
+
+ it("routes below the rollout percentage, shadows in the next band, defaults above", () => {
+ withConfig(
+ {
+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 10,
+ EXTRACTION_PROVIDER_SHADOW_PERCENT: 10,
+ EXTRACTION_PROVIDER_TEAM_IDS: undefined,
+ },
+ () => {
+ const teams = Array.from({ length: 500 }, (_, i) => `team-${i}`);
+ for (const teamId of teams) {
+ const b = cohortBucket(teamId);
+ const expected = b < 10 ? "provider" : b < 20 ? "shadow" : "default";
+ expect(selectExtractionRoute({ ...base, teamId })).toBe(expected);
+ }
+ },
+ );
+ });
+
+ it("routes 100% and 0% as expected", () => {
+ withConfig(
+ {
+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,
+ EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,
+ },
+ () => expect(selectExtractionRoute(base)).toBe("provider"),
+ );
+ withConfig(
+ {
+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,
+ EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,
+ EXTRACTION_PROVIDER_TEAM_IDS: undefined,
+ },
+ () => expect(selectExtractionRoute(base)).toBe("default"),
+ );
+ });
+
+ it("keeps zero-data-retention requests, oversized inputs and unknown teams on the default model", () => {
+ withConfig(
+ {
+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,
+ EXTRACTION_PROVIDER_MAX_INPUT_BYTES: 400_000,
+ },
+ () => {
+ expect(
+ selectExtractionRoute({ ...base, zeroDataRetention: true }),
+ ).toBe("default");
+ expect(selectExtractionRoute({ ...base, inputBytes: 400_001 })).toBe(
+ "default",
+ );
+ expect(selectExtractionRoute({ ...base, teamId: undefined })).toBe(
+ "default",
+ );
+ },
+ );
+ });
+
+ it("always routes allowlisted teams", () => {
+ withConfig(
+ {
+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,
+ EXTRACTION_PROVIDER_TEAM_IDS: "x, team-a ,y",
+ },
+ () => {
+ expect(selectExtractionRoute(base)).toBe("provider");
+ expect(selectExtractionRoute({ ...base, teamId: "team-b" })).toBe(
+ "default",
+ );
+ },
+ );
+ });
+
+ it("prices only the configured provider model", () => {
+ withConfig(
+ {
+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
+ EXTRACTION_PROVIDER_MODEL: "extraction-v1",
+ EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK: 0.07,
+ EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK: 0.2,
+ },
+ () => {
+ expect(extractionProviderCost("extraction-v1")).toEqual({
+ input_cost: 0.07,
+ output_cost: 0.2,
+ });
+ expect(extractionProviderCost("gpt-4o-mini")).toBeUndefined();
+ },
+ );
+ });
+
+ it("honours a requested model for listed teams, or for everyone with the opt-in switch", () => {
+ const cfg = {
+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
+ EXTRACTION_PROVIDER_MODEL: "extraction-v1",
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,
+ EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,
+ };
+ withConfig(
+ {
+ ...cfg,
+ EXTRACTION_PROVIDER_TEAM_IDS: "team-a",
+ EXTRACTION_PROVIDER_REQUEST_OPT_IN: false,
+ },
+ () => {
+ expect(
+ selectExtractionRoute({ ...base, requestedModel: "extraction-v1" }),
+ ).toBe("provider");
+ expect(
+ selectExtractionRoute({
+ ...base,
+ teamId: "team-b",
+ requestedModel: "extraction-v1",
+ }),
+ ).toBe("default");
+ expect(
+ selectExtractionRoute({ ...base, requestedModel: "something-else" }),
+ ).toBe("default");
+ expect(
+ selectExtractionRoute({
+ ...base,
+ requestedModel: "extraction-v1",
+ zeroDataRetention: true,
+ }),
+ ).toBe("default");
+ },
+ );
+ withConfig(
+ {
+ ...cfg,
+ EXTRACTION_PROVIDER_TEAM_IDS: undefined,
+ EXTRACTION_PROVIDER_REQUEST_OPT_IN: true,
+ },
+ () => {
+ expect(
+ selectExtractionRoute({
+ ...base,
+ teamId: "team-b",
+ requestedModel: "extraction-v1",
+ }),
+ ).toBe("provider");
+ },
+ );
+ withConfig({ ...cfg }, () => {
+ expect(isRequestedModelSupported("extraction-v1")).toBe(true);
+ expect(isRequestedModelSupported("gpt-4o-mini")).toBe(false);
+ });
+ withConfig({ ...cfg, EXTRACTION_PROVIDER_BASE_URL: undefined }, () =>
+ expect(isRequestedModelSupported("extraction-v1")).toBe(false),
+ );
+ });
+});
diff --git a/apps/api/src/lib/extraction-provider.ts b/apps/api/src/lib/extraction-provider.ts
new file mode 100644
index 000000000..32ee13c93
--- /dev/null
+++ b/apps/api/src/lib/extraction-provider.ts
@@ -0,0 +1,86 @@
+import { createHash } from "crypto";
+import { config } from "../config";
+
+/**
+ * Routing for the optional JSON extraction provider (EXTRACTION_PROVIDER_*).
+ *
+ * A team lands in a stable cohort bucket 0-99 (sha1 of the team id), so a
+ * rollout percentage selects the same teams every time and can be widened
+ * without reshuffling. Buckets below ROLLOUT_PERCENT are routed; the next
+ * SHADOW_PERCENT buckets are mirrored (the provider is called, the result is
+ * discarded). Teams listed in TEAM_IDS are always routed. Zero-data-retention
+ * requests and inputs over MAX_INPUT_BYTES stay on the default model.
+ *
+ * A request may also name the provider model explicitly (json format `model`);
+ * that routes it regardless of cohort when the team is listed in TEAM_IDS or
+ * REQUEST_OPT_IN is on. Unknown model names are rejected at validation.
+ */
+type ExtractionRoute = "provider" | "shadow" | "default";
+
+function isExtractionProviderConfigured(): boolean {
+ return !!config.EXTRACTION_PROVIDER_BASE_URL;
+}
+
+export function cohortBucket(teamId: string): number {
+ const digest = createHash("sha1").update(teamId).digest();
+ return digest.readUInt32BE(0) % 100;
+}
+
+function allowlistedTeams(): Set<string> {
+ return new Set(
+ (config.EXTRACTION_PROVIDER_TEAM_IDS ?? "")
+ .split(",")
+ .map(s => s.trim())
+ .filter(Boolean),
+ );
+}
+
+/** Model names a request may ask for in the json format `model` field. */
+export function isRequestedModelSupported(model: string): boolean {
+ return (
+ isExtractionProviderConfigured() &&
+ model === config.EXTRACTION_PROVIDER_MODEL
+ );
+}
+
+export function selectExtractionRoute(input: {
+ teamId: string | undefined;
+ zeroDataRetention: boolean;
+ inputBytes: number;
+ requestedModel?: string;
+}): ExtractionRoute {
+ if (!isExtractionProviderConfigured()) return "default";
+ if (input.zeroDataRetention) return "default";
+ if (input.inputBytes > config.EXTRACTION_PROVIDER_MAX_INPUT_BYTES)
+ return "default";
+ if (!input.teamId) return "default";
+ const listed = allowlistedTeams().has(input.teamId);
+ if (input.requestedModel !== undefined) {
+ return isRequestedModelSupported(input.requestedModel) &&
+ (listed || config.EXTRACTION_PROVIDER_REQUEST_OPT_IN)
+ ? "provider"
+ : "default";
+ }
+ if (listed) return "provider";
+ const bucket = cohortBucket(input.teamId);
+ if (bucket < config.EXTRACTION_PROVIDER_ROLLOUT_PERCENT) return "provider";
+ if (
+ bucket <
+ config.EXTRACTION_PROVIDER_ROLLOUT_PERCENT +
+ config.EXTRACTION_PROVIDER_SHADOW_PERCENT
+ )
+ return "shadow";
+ return "default";
+}
+
+/** Per-million-token prices for the provider model (amortized infrastructure cost, configured). */
+export function extractionProviderCost(
+ model: string,
+): { input_cost: number; output_cost: number } | undefined {
+ if (!isExtractionProviderConfigured()) return undefined;
+ if (model !== config.EXTRACTION_PROVIDER_MODEL) return undefined;
+ return {
+ input_cost: config.EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK,
+ output_cost: config.EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK,
+ };
+}
diff --git a/apps/api/src/lib/generic-ai.ts b/apps/api/src/lib/generic-ai.ts
index 90cf89596..8bb2b9d6a 100644
--- a/apps/api/src/lib/generic-ai.ts
+++ b/apps/api/src/lib/generic-ai.ts
@@ -19,8 +19,11 @@ type Provider =
| "openrouter"
| "fireworks"
| "deepinfra"
- | "vertex";
-const defaultProvider: Provider = config.OLLAMA_BASE_URL ? "ollama" : "openai";
+ | "vertex"
+ | "extraction";
+const defaultProvider: Exclude<Provider, "extraction"> = config.OLLAMA_BASE_URL
+ ? "ollama"
+ : "openai";
const providerList: Record<Provider, any> = {
openai: createOpenAI({
@@ -38,6 +41,12 @@ const providerList: Record<Provider, any> = {
}),
fireworks, //FIREWORKS_API_KEY
deepinfra, //DEEPINFRA_API_KEY
+ // OpenAI-compatible JSON extraction provider (EXTRACTION_PROVIDER_*); Chat
+ // Completions only, see getModel.
+ extraction: createOpenAI({
+ apiKey: config.EXTRACTION_PROVIDER_API_KEY ?? "unused",
+ baseURL: config.EXTRACTION_PROVIDER_BASE_URL,
+ }),
vertex: createVertex({
project: "firecrawl",
//https://github.com/vercel/ai/issues/6644 bug
@@ -69,12 +78,17 @@ export function getModel(
if (provider === "openai" && modelName.startsWith("o3-mini")) {
return withUsageTelemetry(providerList.openai.chat(modelName));
}
+ // The extraction provider speaks Chat Completions, not the Responses API,
+ // and is not subject to the MODEL_NAME override.
+ if (provider === "extraction") {
+ return withUsageTelemetry(providerList.extraction.chat(name));
+ }
return withUsageTelemetry(providerList[provider](modelName));
}
export function getEmbeddingModel(
name: string,
- provider: Provider = defaultProvider,
+ provider: Exclude<Provider, "extraction"> = defaultProvider,
) {
return config.MODEL_EMBEDDING_NAME
? providerList[provider].embedding(config.MODEL_EMBEDDING_NAME)
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 4a45b5085..36f501a0f 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -31,6 +31,11 @@ import { CostTracking } from "../../../lib/cost-tracking";
import { isZeroDataRetentionActive } from "../../../lib/otel-tracer";
import { isAgentExtractModelValid } from "../../../controllers/v1/types";
import { hasFormatOfType } from "../../../lib/format-utils";
+import {
+ extractionProviderCost,
+ selectExtractionRoute,
+} from "../../../lib/extraction-provider";
+import { config } from "../../../config";
// Smart model selection based on schema
function detectRecursiveSchema(schema: any): boolean {
@@ -307,6 +312,10 @@ function lookupModelCost(
const slash = model.indexOf("/");
if (slash !== -1) candidates.push(model.slice(slash + 1));
+ // The configured extraction provider is priced from config even if its
+ // name collides with a static entry.
+ const providerCost = extractionProviderCost(model);
+ if (providerCost) return providerCost;
for (const id of candidates) {
if (modelCosts[id]) return modelCosts[id];
}
@@ -1057,9 +1066,27 @@ export async function performLLMExtract(
zeroDataRetention: meta.internalOptions.zeroDataRetention,
};
- const { extractedDataArray, warning, costLimitExceededTokenUsage } =
- await extractData({
- extractOptions: generationOptions,
+ // Optional extraction provider: routed by team cohort, with the default
+ // model as a per-request fallback; or mirrored (shadow) without using
+ // the result. The agent path is never routed.
+ const route = useAgent
+ ? "default"
+ : selectExtractionRoute({
+ teamId: meta.internalOptions.teamId,
+ zeroDataRetention:
+ !!meta.internalOptions.zeroDataRetention ||
+ isZeroDataRetentionActive(),
+ inputBytes: Buffer.byteLength(document.markdown ?? "", "utf8"),
+ requestedModel: jsonFormat.model,
+ });
+ if (jsonFormat.model !== undefined && route !== "provider") {
+ document.warning =
+ "The requested extraction model is not available for this request; the default model was used." +
+ (document.warning ? " " + document.warning : "");
+ }
+ const extractWith = (options: GenerateCompletionsOptions) =>
+ extractData({
+ extractOptions: options,
urls: [meta.rewrittenUrl ?? meta.url],
useAgent,
checkPromptInjection: meta.options.checkPromptInjection,
@@ -1069,6 +1096,63 @@ export async function performLLMExtract(
functionId: llmFunctionId,
},
});
+ const providerOptions = (): GenerateCompletionsOptions => ({
+ ...generationOptions,
+ model: getModel(config.EXTRACTION_PROVIDER_MODEL, "extraction"),
+ retryModel: getModel(modelSelection.modelName, "openai"),
+ });
+
+ // extractData reports generation failures as a warning rather than
+ // throwing, so both paths count as a provider failure.
+ const PROVIDER_FAILURE = "JSON extraction failed:";
+ let extraction: Awaited<ReturnType<typeof extractWith>>;
+ if (route === "provider") {
+ let attempt: Awaited<ReturnType<typeof extractWith>> | undefined;
+ let failure: string | undefined;
+ try {
+ attempt = await extractWith(providerOptions());
+ if (attempt.warning?.includes(PROVIDER_FAILURE)) {
+ failure = attempt.warning;
+ }
+ } catch (error) {
+ failure = (error as Error).message;
+ }
+ if (attempt !== undefined && failure === undefined) {
+ extraction = attempt;
+ } else {
+ meta.logger.warn(
+ "Extraction provider failed, falling back to the default model",
+ { error: failure },
+ );
+ extraction = await extractWith(generationOptions);
+ }
+ } else {
+ extraction = await extractWith(generationOptions);
+ if (route === "shadow") {
+ // Fire and forget: the provider's own telemetry records the outcome;
+ // nothing is billed to the team and nothing changes in the response.
+ void extractWith({
+ ...providerOptions(),
+ costTrackingOptions: {
+ costTracking: new CostTracking(),
+ metadata: {
+ module: "scrapeURL",
+ method: "performLLMExtract/shadow",
+ },
+ },
+ metadata: {
+ ...generationOptions.metadata,
+ functionId: "performLLMExtract/shadow",
+ },
+ }).catch(error => {
+ meta.logger.debug("Extraction provider shadow call failed", {
+ error: (error as Error).message,
+ });
+ });
+ }
+ }
+ const { extractedDataArray, warning, costLimitExceededTokenUsage } =
+ extraction;
if (warning) {
document.warning =