Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

chore(api): remove extraction provider (#5730)

Abimael Martellace3c58bc88f
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 620793d0b..bafd27e5e 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -82,22 +82,6 @@ TYPESAFE_API_KEY=
# BRANDING_JEV_TIMEOUT_MS=5000
# Share of teams (0-100) whose branding scrapes also show Jev the page's first screen, as if they set the branding format's __experimental_screenshot
# BRANDING_JEV_SCREENSHOT_PERCENT=0
-# optional OpenAI-compatible provider for JSON extraction; routed by team cohort
-# at ROLLOUT_PERCENT, mirrored at SHADOW_PERCENT, with per-request fallback
-EXTRACTION_PROVIDER_BASE_URL=
-EXTRACTION_PROVIDER_API_KEY=
-EXTRACTION_PROVIDER_MODEL=extraction-v1
-EXTRACTION_PROVIDER_ROLLOUT_PERCENT=0
-EXTRACTION_PROVIDER_SHADOW_PERCENT=0
-# EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT= # rollout for schemas with $ref/$defs; unset = ROLLOUT_PERCENT
-# EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS=true # all-empty-arrays answers on long pages fall back to the default model
-EXTRACTION_PROVIDER_TEAM_IDS=
-# EXTRACTION_PROVIDER_REQUEST_OPT_IN=false
-# inputs above this many bytes stay on the default model
-# EXTRACTION_PROVIDER_MAX_INPUT_BYTES=400000
-# amortized price of the provider model, per million tokens, for cost tracking
-# EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK=0.07
-# EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK=0.2
BULL_AUTH_KEY=@
# optional second bearer value accepted on /admin/ops/* while callers move to a new BULL_AUTH_KEY
# BULL_AUTH_KEY_PREVIOUS=
diff --git a/apps/api/src/__tests__/snips/v2/extraction-provider.test.ts b/apps/api/src/__tests__/snips/v2/extraction-provider.test.ts
deleted file mode 100644
index 7d278f5ad..000000000
--- a/apps/api/src/__tests__/snips/v2/extraction-provider.test.ts
+++ /dev/null
@@ -1,84 +0,0 @@
-import {
- scrape,
- scrapeWithFailure,
- scrapeTimeout,
- idmux,
- Identity,
-} from "./lib";
-
-// Runs only where the API under test is guaranteed to route to the provider:
-// a configured base URL and either a 100% rollout or the per-request opt-in.
-const configured =
- !!process.env.EXTRACTION_PROVIDER_BASE_URL &&
- (process.env.EXTRACTION_PROVIDER_ROLLOUT_PERCENT === "100" ||
- ["true", "1"].includes(
- (process.env.EXTRACTION_PROVIDER_REQUEST_OPT_IN ?? "").toLowerCase(),
- ));
-
-let identity: Identity;
-
-beforeAll(async () => {
- identity = await idmux({
- name: "extraction-provider",
- concurrency: 10,
- credits: 1_000_000,
- });
-}, 10000);
-
-(configured ? describe : describe.skip)(
- "JSON extraction via the extraction provider",
- () => {
- it.concurrent(
- "returns a schema-conforming object",
- async () => {
- const response = await scrape(
- {
- url: "https://firecrawl.dev",
- formats: [
- {
- type: "json",
- schema: {
- type: "object",
- properties: {
- company_name: { type: "string" },
- tagline: { type: "string" },
- },
- required: ["company_name", "tagline"],
- },
- },
- ],
- },
- identity,
- );
- expect(response.json).toBeDefined();
- expect(typeof response.json.company_name).toBe("string");
- expect(typeof response.json.tagline).toBe("string");
- expect(response.json.company_name.toLowerCase()).toContain("firecrawl");
- },
- scrapeTimeout,
- );
- },
-);
-
-describe("JSON format model option", () => {
- it.concurrent(
- "rejects a model that is not available",
- async () => {
- const response = await scrapeWithFailure(
- {
- url: "https://firecrawl.dev",
- formats: [
- {
- type: "json",
- model: "not-a-configured-extraction-model",
- schema: { type: "object", properties: { x: { type: "string" } } },
- },
- ],
- },
- identity,
- );
- expect(response.error).toContain("Unsupported extraction model");
- },
- scrapeTimeout,
- );
-});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index bd21dfdae..6a2e7fcec 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -717,56 +717,6 @@ const configSchema = z.object({
// AI/ML
MODEL_NAME: z.string().optional(),
- // Optional OpenAI-compatible provider for JSON extraction. Traffic is routed
- // by team cohort (stable hash) at ROLLOUT_PERCENT, mirrored without using the
- // result at SHADOW_PERCENT, and always routed for TEAM_IDS. Any failure falls
- // back to the default model for that request.
- EXTRACTION_PROVIDER_BASE_URL: blankStringAsUnset(z.string().url().optional()),
- EXTRACTION_PROVIDER_API_KEY: blankStringAsUnset(z.string().optional()),
- EXTRACTION_PROVIDER_MODEL: blankStringAsUnset(
- z.string().min(1).default("extraction-v1"),
- ),
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: z.coerce
- .number()
- .int()
- .min(0)
- .max(100)
- .default(0),
- EXTRACTION_PROVIDER_SHADOW_PERCENT: z.coerce
- .number()
- .int()
- .min(0)
- .max(100)
- .default(0),
- // Rollout percentage for schemas with $ref/$defs (the ones the default
- // path sends to its larger model); unset = same as ROLLOUT_PERCENT.
- EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: blankStringAsUnset(
- z.coerce.number().int().min(0).max(100).optional(),
- ),
- // A provider answer whose arrays are all empty, for a page long enough to
- // plausibly hold items, counts as a failure and falls back to the default
- // model for that request.
- EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: z
- .stringbool()
- .optional()
- .default(true),
- EXTRACTION_PROVIDER_TEAM_IDS: z.string().optional(),
- // Let any team pick the provider per request (json format `model`); off =
- // only teams in TEAM_IDS may.
- EXTRACTION_PROVIDER_REQUEST_OPT_IN: z.stringbool().optional(),
- EXTRACTION_PROVIDER_MAX_INPUT_BYTES: z.coerce
- .number()
- .int()
- .nonnegative()
- .default(400_000),
- EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK: z.coerce
- .number()
- .nonnegative()
- .default(0.07),
- EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK: z.coerce
- .number()
- .nonnegative()
- .default(0.2),
MODEL_EMBEDDING_NAME: z.string().optional(),
OLLAMA_BASE_URL: z.string().optional(),
VERTEX_CREDENTIALS: z.string().optional(),
@@ -889,22 +839,6 @@ const configSchema = z.object({
});
const validatedConfigSchema = configSchema.superRefine((value, context) => {
- for (const key of [
- "EXTRACTION_PROVIDER_ROLLOUT_PERCENT",
- "EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT",
- ] as const) {
- const rollout = value[key];
- if (
- rollout !== undefined &&
- rollout + value.EXTRACTION_PROVIDER_SHADOW_PERCENT > 100
- ) {
- context.addIssue({
- code: "custom",
- path: ["EXTRACTION_PROVIDER_SHADOW_PERCENT"],
- message: `${key} + EXTRACTION_PROVIDER_SHADOW_PERCENT must not exceed 100`,
- });
- }
- }
if (value.GOV_ROUTING_ENABLED && !value.SEARCH_PLATFORM_URL) {
context.addIssue({
code: "custom",
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index bc15c3b0b..6a387357f 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -3,7 +3,6 @@ import type { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";
import { Request, Response } from "express";
import { hasCategory } from "../../lib/search-query-builder";
import { config } from "../../config";
-import { isRequestedModelSupported } from "../../lib/extraction-provider";
import { z } from "zod";
import { browserProfileNameSchema } from "../../lib/browser-profiles";
import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
@@ -367,14 +366,6 @@ const jsonFormatWithOptions = z.strictObject({
})
.superRefine(addStrictSchemaIssue),
prompt: z.string().max(10000).optional(),
- // Optional: name a configured extraction model for this request.
- model: z
- .string()
- .max(100)
- .optional()
- .refine(val => val === undefined || isRequestedModelSupported(val), {
- message: "Unsupported extraction model.",
- }),
// Deprecated: lifted into the top-level checkPromptInjection at parse time.
checkPromptInjection: z.boolean().optional(),
});
diff --git a/apps/api/src/lib/extraction-provider.test.ts b/apps/api/src/lib/extraction-provider.test.ts
deleted file mode 100644
index 9a9371a6c..000000000
--- a/apps/api/src/lib/extraction-provider.test.ts
+++ /dev/null
@@ -1,278 +0,0 @@
-import { config } from "../config";
-import {
- cohortBucket,
- extractionProviderCost,
- isEmptyListAnswer,
- isRequestedModelSupported,
- selectExtractionRoute,
-} from "./extraction-provider";
-
-const base = { teamId: "team-a", zeroDataRetention: false, inputBytes: 10_000 };
-
-function withConfig<T>(overrides: Partial<typeof config>, fn: () => T): T {
- const saved: Partial<typeof config> = {};
- for (const k of Object.keys(overrides) as (keyof typeof config)[]) {
- (saved as any)[k] = config[k];
- (config as any)[k] = overrides[k];
- }
- try {
- return fn();
- } finally {
- Object.assign(config, saved);
- }
-}
-
-describe("extraction provider routing", () => {
- it("is off when no provider is configured", () => {
- withConfig(
- {
- EXTRACTION_PROVIDER_BASE_URL: undefined,
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,
- },
- () => expect(selectExtractionRoute(base)).toBe("default"),
- );
- });
-
- it("buckets teams stably and evenly enough to roll out by percentage", () => {
- const a = cohortBucket("team-a");
- expect(cohortBucket("team-a")).toBe(a);
- expect(a).toBeGreaterThanOrEqual(0);
- expect(a).toBeLessThan(100);
- const buckets = new Set(
- Array.from({ length: 2000 }, (_, i) => cohortBucket(`team-${i}`)),
- );
- expect(buckets.size).toBe(100);
- });
-
- it("routes below the rollout percentage, shadows in the next band, defaults above", () => {
- withConfig(
- {
- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 10,
- EXTRACTION_PROVIDER_SHADOW_PERCENT: 10,
- EXTRACTION_PROVIDER_TEAM_IDS: undefined,
- },
- () => {
- const teams = Array.from({ length: 500 }, (_, i) => `team-${i}`);
- for (const teamId of teams) {
- const b = cohortBucket(teamId);
- const expected = b < 10 ? "provider" : b < 20 ? "shadow" : "default";
- expect(selectExtractionRoute({ ...base, teamId })).toBe(expected);
- }
- },
- );
- });
-
- it("routes 100% and 0% as expected", () => {
- withConfig(
- {
- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,
- EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,
- },
- () => expect(selectExtractionRoute(base)).toBe("provider"),
- );
- withConfig(
- {
- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,
- EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,
- EXTRACTION_PROVIDER_TEAM_IDS: undefined,
- },
- () => expect(selectExtractionRoute(base)).toBe("default"),
- );
- });
-
- it("keeps zero-data-retention requests, oversized inputs and unknown teams on the default model", () => {
- withConfig(
- {
- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,
- EXTRACTION_PROVIDER_MAX_INPUT_BYTES: 400_000,
- },
- () => {
- expect(
- selectExtractionRoute({ ...base, zeroDataRetention: true }),
- ).toBe("default");
- expect(selectExtractionRoute({ ...base, inputBytes: 400_001 })).toBe(
- "default",
- );
- expect(selectExtractionRoute({ ...base, teamId: undefined })).toBe(
- "default",
- );
- },
- );
- });
-
- it("always routes allowlisted teams", () => {
- withConfig(
- {
- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,
- EXTRACTION_PROVIDER_TEAM_IDS: "x, team-a ,y",
- },
- () => {
- expect(selectExtractionRoute(base)).toBe("provider");
- expect(selectExtractionRoute({ ...base, teamId: "team-b" })).toBe(
- "default",
- );
- },
- );
- });
-
- it("prices only the configured provider model", () => {
- withConfig(
- {
- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
- EXTRACTION_PROVIDER_MODEL: "extraction-v1",
- EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK: 0.07,
- EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK: 0.2,
- },
- () => {
- expect(extractionProviderCost("extraction-v1")).toEqual({
- input_cost: 0.07,
- output_cost: 0.2,
- });
- expect(extractionProviderCost("gpt-4o-mini")).toBeUndefined();
- },
- );
- });
-
- it("honours a requested model for listed teams, or for everyone with the opt-in switch", () => {
- const cfg = {
- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
- EXTRACTION_PROVIDER_MODEL: "extraction-v1",
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,
- EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,
- };
- withConfig(
- {
- ...cfg,
- EXTRACTION_PROVIDER_TEAM_IDS: "team-a",
- EXTRACTION_PROVIDER_REQUEST_OPT_IN: false,
- },
- () => {
- expect(
- selectExtractionRoute({ ...base, requestedModel: "extraction-v1" }),
- ).toBe("provider");
- expect(
- selectExtractionRoute({
- ...base,
- teamId: "team-b",
- requestedModel: "extraction-v1",
- }),
- ).toBe("default");
- expect(
- selectExtractionRoute({ ...base, requestedModel: "something-else" }),
- ).toBe("default");
- expect(
- selectExtractionRoute({
- ...base,
- requestedModel: "extraction-v1",
- zeroDataRetention: true,
- }),
- ).toBe("default");
- },
- );
- withConfig(
- {
- ...cfg,
- EXTRACTION_PROVIDER_TEAM_IDS: undefined,
- EXTRACTION_PROVIDER_REQUEST_OPT_IN: true,
- },
- () => {
- expect(
- selectExtractionRoute({
- ...base,
- teamId: "team-b",
- requestedModel: "extraction-v1",
- }),
- ).toBe("provider");
- },
- );
- withConfig({ ...cfg }, () => {
- expect(isRequestedModelSupported("extraction-v1")).toBe(true);
- expect(isRequestedModelSupported("gpt-4o-mini")).toBe(false);
- });
- withConfig({ ...cfg, EXTRACTION_PROVIDER_BASE_URL: undefined }, () =>
- expect(isRequestedModelSupported("extraction-v1")).toBe(false),
- );
- });
-});
-
-describe("ref-schema rollout", () => {
- const provider = {
- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
- EXTRACTION_PROVIDER_TEAM_IDS: undefined,
- EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,
- };
-
- it("uses REF_SCHEMA_ROLLOUT_PERCENT for ref schemas and ROLLOUT_PERCENT otherwise", () => {
- withConfig(
- {
- ...provider,
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,
- EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: 100,
- },
- () => {
- expect(selectExtractionRoute({ ...base, refSchema: true })).toBe(
- "provider",
- );
- expect(selectExtractionRoute({ ...base, refSchema: false })).toBe(
- "default",
- );
- expect(selectExtractionRoute(base)).toBe("default");
- },
- );
- });
-
- it("falls back to ROLLOUT_PERCENT when the ref percentage is unset", () => {
- withConfig(
- {
- ...provider,
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,
- EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: undefined,
- },
- () =>
- expect(selectExtractionRoute({ ...base, refSchema: true })).toBe(
- "provider",
- ),
- );
- });
-
- it("keeps the shadow band directly above the ref rollout band", () => {
- withConfig(
- {
- ...provider,
- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,
- EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: 20,
- EXTRACTION_PROVIDER_SHADOW_PERCENT: 10,
- },
- () => {
- for (let i = 0; i < 300; i++) {
- const teamId = `team-${i}`;
- const b = cohortBucket(teamId);
- expect(
- selectExtractionRoute({ ...base, teamId, refSchema: true }),
- ).toBe(b < 20 ? "provider" : b < 30 ? "shadow" : "default");
- expect(selectExtractionRoute({ ...base, teamId })).toBe(
- b < 10 ? "shadow" : "default",
- );
- }
- },
- );
- });
-});
-
-describe("isEmptyListAnswer", () => {
- it("is true only when the answer has arrays and all of them are empty", () => {
- expect(isEmptyListAnswer({ jobs: [] })).toBe(true);
- expect(isEmptyListAnswer({ page: { items: [], tags: [] } })).toBe(true);
- expect(isEmptyListAnswer([])).toBe(true);
- expect(isEmptyListAnswer({ jobs: [{ title: "Cook" }] })).toBe(false);
- expect(isEmptyListAnswer({ jobs: [], seen: [1] })).toBe(false);
- expect(isEmptyListAnswer({ title: "No lists here" })).toBe(false);
- expect(isEmptyListAnswer(undefined)).toBe(false);
- expect(isEmptyListAnswer(null)).toBe(false);
- });
-});
diff --git a/apps/api/src/lib/extraction-provider.ts b/apps/api/src/lib/extraction-provider.ts
deleted file mode 100644
index 971f54512..000000000
--- a/apps/api/src/lib/extraction-provider.ts
+++ /dev/null
@@ -1,119 +0,0 @@
-import { createHash } from "crypto";
-import { config } from "../config";
-
-/**
- * Routing for the optional JSON extraction provider (EXTRACTION_PROVIDER_*).
- *
- * A team lands in a stable cohort bucket 0-99 (sha1 of the team id), so a
- * rollout percentage selects the same teams every time and can be widened
- * without reshuffling. Buckets below ROLLOUT_PERCENT are routed; the next
- * SHADOW_PERCENT buckets are mirrored (the provider is called, the result is
- * discarded). Teams listed in TEAM_IDS are always routed. Zero-data-retention
- * requests and inputs over MAX_INPUT_BYTES stay on the default model.
- *
- * A request may also name the provider model explicitly (json format `model`);
- * that routes it regardless of cohort when the team is listed in TEAM_IDS or
- * REQUEST_OPT_IN is on. Unknown model names are rejected at validation.
- *
- * Schemas with $ref/$defs can roll out on their own percentage
- * (REF_SCHEMA_ROLLOUT_PERCENT), since the default path prices them
- * differently. An answer that is nothing but empty arrays on a page long
- * enough to hold items is treated as a provider failure (see
- * isEmptyListAnswer) so the request falls back instead of returning nothing.
- */
-type ExtractionRoute = "provider" | "shadow" | "default";
-
-function isExtractionProviderConfigured(): boolean {
- return !!config.EXTRACTION_PROVIDER_BASE_URL;
-}
-
-export function cohortBucket(teamId: string): number {
- const digest = createHash("sha1").update(teamId).digest();
- return digest.readUInt32BE(0) % 100;
-}
-
-function allowlistedTeams(): Set<string> {
- return new Set(
- (config.EXTRACTION_PROVIDER_TEAM_IDS ?? "")
- .split(",")
- .map(s => s.trim())
- .filter(Boolean),
- );
-}
-
-/** Model names a request may ask for in the json format `model` field. */
-export function isRequestedModelSupported(model: string): boolean {
- return (
- isExtractionProviderConfigured() &&
- model === config.EXTRACTION_PROVIDER_MODEL
- );
-}
-
-export function selectExtractionRoute(input: {
- teamId: string | undefined;
- zeroDataRetention: boolean;
- inputBytes: number;
- requestedModel?: string;
- /** The schema uses $ref/$defs (routed by REF_SCHEMA_ROLLOUT_PERCENT when set). */
- refSchema?: boolean;
-}): ExtractionRoute {
- if (!isExtractionProviderConfigured()) return "default";
- if (input.zeroDataRetention) return "default";
- if (input.inputBytes > config.EXTRACTION_PROVIDER_MAX_INPUT_BYTES)
- return "default";
- if (!input.teamId) return "default";
- const listed = allowlistedTeams().has(input.teamId);
- if (input.requestedModel !== undefined) {
- return isRequestedModelSupported(input.requestedModel) &&
- (listed || config.EXTRACTION_PROVIDER_REQUEST_OPT_IN)
- ? "provider"
- : "default";
- }
- if (listed) return "provider";
- const bucket = cohortBucket(input.teamId);
- const rollout =
- input.refSchema &&
- config.EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT !== undefined
- ? config.EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT
- : config.EXTRACTION_PROVIDER_ROLLOUT_PERCENT;
- if (bucket < rollout) return "provider";
- if (bucket < rollout + config.EXTRACTION_PROVIDER_SHADOW_PERCENT)
- return "shadow";
- return "default";
-}
-
-/** Pages shorter than this can legitimately have no list items. */
-export const EMPTY_LIST_FALLBACK_MIN_CHARS = 2000;
-
-function arraysIn(value: unknown, out: unknown[][] = []): unknown[][] {
- if (Array.isArray(value)) {
- out.push(value);
- for (const v of value) arraysIn(v, out);
- } else if (value && typeof value === "object") {
- for (const v of Object.values(value as Record<string, unknown>))
- arraysIn(v, out);
- }
- return out;
-}
-
-/**
- * True when the answer contains arrays and every one of them is empty — the
- * shape a model produces when it gives up on a listing page. Only meaningful
- * with a markdown input of at least EMPTY_LIST_FALLBACK_MIN_CHARS.
- */
-export function isEmptyListAnswer(answer: unknown): boolean {
- const arrays = arraysIn(answer);
- return arrays.length > 0 && arrays.every(a => a.length === 0);
-}
-
-/** Per-million-token prices for the provider model (amortized infrastructure cost, configured). */
-export function extractionProviderCost(
- model: string,
-): { input_cost: number; output_cost: number } | undefined {
- if (!isExtractionProviderConfigured()) return undefined;
- if (model !== config.EXTRACTION_PROVIDER_MODEL) return undefined;
- return {
- input_cost: config.EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK,
- output_cost: config.EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK,
- };
-}
diff --git a/apps/api/src/lib/generic-ai.ts b/apps/api/src/lib/generic-ai.ts
index 24f6922cf..26c9d9e6c 100644
--- a/apps/api/src/lib/generic-ai.ts
+++ b/apps/api/src/lib/generic-ai.ts
@@ -23,11 +23,8 @@ type Provider =
| "openrouter"
| "fireworks"
| "deepinfra"
- | "vertex"
- | "extraction";
-const defaultProvider: Exclude<Provider, "extraction"> = config.OLLAMA_BASE_URL
- ? "ollama"
- : "openai";
+ | "vertex";
+const defaultProvider: Provider = config.OLLAMA_BASE_URL ? "ollama" : "openai";
const vertexAuthOptions = config.VERTEX_CREDENTIALS
? {
@@ -53,12 +50,6 @@ const providerList: Record<Provider, any> = {
}),
fireworks, //FIREWORKS_API_KEY
deepinfra, //DEEPINFRA_API_KEY
- // OpenAI-compatible JSON extraction provider (EXTRACTION_PROVIDER_*); Chat
- // Completions only, see getModel.
- extraction: createOpenAI({
- apiKey: config.EXTRACTION_PROVIDER_API_KEY ?? "unused",
- baseURL: config.EXTRACTION_PROVIDER_BASE_URL,
- }),
vertex: createVertex({
project: "firecrawl",
//https://github.com/vercel/ai/issues/6644 bug
@@ -164,17 +155,12 @@ export function getModel(
if (provider === "openai" && modelName.startsWith("o3-mini")) {
return withUsageTelemetry(providerList.openai.chat(modelName));
}
- // The extraction provider speaks Chat Completions, not the Responses API,
- // and is not subject to the MODEL_NAME override.
- if (provider === "extraction") {
- return withUsageTelemetry(providerList.extraction.chat(name));
- }
return withUsageTelemetry(providerList[provider](modelName));
}
export function getEmbeddingModel(
name: string,
- provider: Exclude<Provider, "extraction"> = defaultProvider,
+ provider: Provider = defaultProvider,
options?: Pick<ModelOptions, "teamFlags" | "feature">,
) {
const modelName = config.MODEL_EMBEDDING_NAME || name;
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
index 4d02ab07a..25950002b 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
@@ -19,7 +19,6 @@ import { performSummary } from "./llmExtract";
import { performCleanContent } from "./llmExtract";
import { encoding_for_model } from "@dqbd/tiktoken";
import type { Mock } from "vitest";
-import { config } from "../../../config";
vi.mock("@dqbd/tiktoken", () => ({
encoding_for_model: vi.fn(),
@@ -483,127 +482,3 @@ describe("performLLMExtract empty-result handling", () => {
);
});
});
-
-describe("performLLMExtract extraction provider fallback", () => {
- const listSchema = {
- type: "object",
- properties: {
- jobs: {
- type: "array",
- items: { type: "object", properties: { title: { type: "string" } } },
- },
- },
- };
- const longPage = "Open roles: Line cook. Pizza cook. Apply now. ".repeat(60); // > 2,000 chars
- const saved: Partial<typeof config> = {};
-
- function withProvider(overrides: Partial<typeof config> = {}) {
- const values: Partial<typeof config> = {
- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
- EXTRACTION_PROVIDER_MODEL: "extraction-v1",
- EXTRACTION_PROVIDER_TEAM_IDS: "test-team",
- EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: true,
- ...overrides,
- };
- for (const k of Object.keys(values) as (keyof typeof config)[]) {
- (saved as any)[k] = config[k];
- (config as any)[k] = values[k];
- }
- }
-
- function meta() {
- const m = makeJsonMeta();
- m.options.formats[0].schema = listSchema as any;
- (m.logger as any).warn = vi.fn(); // the fallback path logs a warning
- return m;
- }
-
- function modelIds() {
- return (extractData as Mock).mock.calls.map(
- c => c[0].extractOptions.model.modelId,
- );
- }
-
- beforeEach(() => {
- (extractData as Mock).mockReset();
- });
-
- afterEach(() => {
- Object.assign(config, saved);
- for (const k of Object.keys(saved)) delete (saved as any)[k];
- });
-
- it("falls back to the default model when the provider answers with only empty lists on a long page", async () => {
- withProvider();
- (extractData as Mock)
- .mockResolvedValueOnce({
- extractedDataArray: [{ jobs: [] }],
- warning: undefined,
- costLimitExceededTokenUsage: null,
- })
- .mockResolvedValueOnce({
- extractedDataArray: [{ jobs: [{ title: "Line cook" }] }],
- warning: undefined,
- costLimitExceededTokenUsage: null,
- });
-
- const result = await performLLMExtract(meta(), {
- markdown: longPage,
- metadata: {},
- } as any);
-
- expect(modelIds()).toEqual(["extraction-v1", "gpt-4o-mini"]);
- expect(result.json).toEqual({ jobs: [{ title: "Line cook" }] });
- });
-
- it("keeps the provider's empty lists on a short page", async () => {
- withProvider();
- (extractData as Mock).mockResolvedValueOnce({
- extractedDataArray: [{ jobs: [] }],
- warning: undefined,
- costLimitExceededTokenUsage: null,
- });
-
- const result = await performLLMExtract(meta(), {
- markdown: "No openings right now.",
- metadata: {},
- } as any);
-
- expect(modelIds()).toEqual(["extraction-v1"]);
- expect(result.json).toEqual({ jobs: [] });
- });
-
- it("keeps the provider's empty lists when the fallback is switched off", async () => {
- withProvider({ EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: false });
- (extractData as Mock).mockResolvedValueOnce({
- extractedDataArray: [{ jobs: [] }],
- warning: undefined,
- costLimitExceededTokenUsage: null,
- });
-
- const result = await performLLMExtract(meta(), {
- markdown: longPage,
- metadata: {},
- } as any);
-
- expect(modelIds()).toEqual(["extraction-v1"]);
- expect(result.json).toEqual({ jobs: [] });
- });
-
- it("does not fall back when the provider found items", async () => {
- withProvider();
- (extractData as Mock).mockResolvedValueOnce({
- extractedDataArray: [{ jobs: [{ title: "Pizza cook" }] }],
- warning: undefined,
- costLimitExceededTokenUsage: null,
- });
-
- const result = await performLLMExtract(meta(), {
- markdown: longPage,
- metadata: {},
- } as any);
-
- expect(modelIds()).toEqual(["extraction-v1"]);
- expect(result.json).toEqual({ jobs: [{ title: "Pizza cook" }] });
- });
-});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 8fe6491fa..bea6849fa 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -32,13 +32,6 @@ import { CostTracking } from "../../../lib/cost-tracking";
import { isZeroDataRetentionActive } from "../../../lib/otel-tracer";
import { isAgentExtractModelValid } from "../../../controllers/v1/types";
import { hasFormatOfType } from "../../../lib/format-utils";
-import {
- EMPTY_LIST_FALLBACK_MIN_CHARS,
- extractionProviderCost,
- isEmptyListAnswer,
- selectExtractionRoute,
-} from "../../../lib/extraction-provider";
-import { config } from "../../../config";
import { isUsOnly } from "../../../lib/us-only";
// Smart model selection based on schema
@@ -316,10 +309,6 @@ function lookupModelCost(
const slash = model.indexOf("/");
if (slash !== -1) candidates.push(model.slice(slash + 1));
- // The configured extraction provider is priced from config even if its
- // name collides with a static entry.
- const providerCost = extractionProviderCost(model);
- if (providerCost) return providerCost;
for (const id of candidates) {
if (modelCosts[id]) return modelCosts[id];
}
@@ -393,8 +382,6 @@ export type GenerateCompletionsOptions = {
isExtractEndpoint?: boolean;
mode?: "object" | "no-object";
providerOptions?: LanguageModelV1ProviderMetadata;
- /** AI SDK retry count for the primary model call (default: the SDK's). */
- maxRetries?: number;
retryModel?: LanguageModel;
costTrackingOptions: {
costTracking: CostTracking;
@@ -464,7 +451,6 @@ export async function generateCompletions({
model = getModel("gpt-4o-mini", "openai", { teamFlags }),
mode = "object",
providerOptions,
- maxRetries,
retryModel = getModel("gpt-4.1-mini", "openai", { teamFlags }),
costTrackingOptions,
metadata,
@@ -794,7 +780,6 @@ export async function generateCompletions({
const generateObjectConfig = {
model: currentModel,
prompt: prompt,
- ...(maxRetries !== undefined && { maxRetries }),
providerOptions: {
...(providerOptions || {}),
google: {
@@ -902,7 +887,6 @@ export async function generateCompletions({
const retryConfig = {
...generateObjectConfig,
model: currentModel,
- maxRetries: undefined, // the fallback model keeps the SDK default
};
result = await generateObject(retryConfig);
costTrackingOptions.costTracking.addCall({
@@ -1094,29 +1078,9 @@ export async function performLLMExtract(
teamFlags,
};
- // Optional extraction provider: routed by team cohort, with the default
- // model as a per-request fallback; or mirrored (shadow) without using
- // the result. The agent path and US-only teams are never routed.
- const route =
- useAgent || usOnly
- ? "default"
- : selectExtractionRoute({
- teamId: meta.internalOptions.teamId,
- zeroDataRetention:
- !!meta.internalOptions.zeroDataRetention ||
- isZeroDataRetentionActive(),
- inputBytes: Buffer.byteLength(document.markdown ?? "", "utf8"),
- requestedModel: jsonFormat.model,
- refSchema: modelSelection.reason === "recursive_schema_detected",
- });
- if (jsonFormat.model !== undefined && route !== "provider") {
- document.warning =
- "The requested extraction model is not available for this request; the default model was used." +
- (document.warning ? " " + document.warning : "");
- }
- const extractWith = (options: GenerateCompletionsOptions) =>
- extractData({
- extractOptions: options,
+ const { extractedDataArray, warning, costLimitExceededTokenUsage } =
+ await extractData({
+ extractOptions: generationOptions,
urls: [meta.rewrittenUrl ?? meta.url],
useAgent,
checkPromptInjection: meta.options.checkPromptInjection,
@@ -1126,74 +1090,6 @@ export async function performLLMExtract(
functionId: llmFunctionId,
},
});
- const providerOptions = (): GenerateCompletionsOptions => ({
- ...generationOptions,
- model: getModel(config.EXTRACTION_PROVIDER_MODEL, "extraction"),
- retryModel: getModel(modelSelection.modelName, "openai"),
- // An unreachable provider should fall back at once, not after the
- // SDK's retries (~6 s of connection attempts per request).
- maxRetries: 0,
- });
-
- // extractData reports generation failures as a warning rather than
- // throwing, so both paths count as a provider failure.
- const PROVIDER_FAILURE = "JSON extraction failed:";
- let extraction: Awaited<ReturnType<typeof extractWith>>;
- if (route === "provider") {
- let attempt: Awaited<ReturnType<typeof extractWith>> | undefined;
- let failure: string | undefined;
- try {
- attempt = await extractWith(providerOptions());
- if (attempt.warning?.includes(PROVIDER_FAILURE)) {
- failure = attempt.warning;
- } else if (
- config.EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS &&
- (document.markdown?.length ?? 0) >= EMPTY_LIST_FALLBACK_MIN_CHARS &&
- isEmptyListAnswer(
- attempt.extractedDataArray[attempt.extractedDataArray.length - 1],
- )
- ) {
- failure = "provider returned only empty lists";
- }
- } catch (error) {
- failure = (error as Error).message;
- }
- if (attempt !== undefined && failure === undefined) {
- extraction = attempt;
- } else {
- meta.logger.warn(
- "Extraction provider failed, falling back to the default model",
- { error: failure },
- );
- extraction = await extractWith(generationOptions);
- }
- } else {
- extraction = await extractWith(generationOptions);
- if (route === "shadow") {
- // Fire and forget: the provider's own telemetry records the outcome;
- // nothing is billed to the team and nothing changes in the response.
- void extractWith({
- ...providerOptions(),
- costTrackingOptions: {
- costTracking: new CostTracking(),
- metadata: {
- module: "scrapeURL",
- method: "performLLMExtract/shadow",
- },
- },
- metadata: {
- ...generationOptions.metadata,
- functionId: "performLLMExtract/shadow",
- },
- }).catch(error => {
- meta.logger.debug("Extraction provider shadow call failed", {
- error: (error as Error).message,
- });
- });
- }
- }
- const { extractedDataArray, warning, costLimitExceededTokenUsage } =
- extraction;
if (warning) {
document.warning =