Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore(api): remove extraction provider (#5730)
Abimael Martellace3c58bc88f
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 620793d0b..bafd27e5e 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -82,22 +82,6 @@ TYPESAFE_API_KEY=# BRANDING_JEV_TIMEOUT_MS=5000# Share of teams (0-100) whose branding scrapes also show Jev the page's first screen, as if they set the branding format's __experimental_screenshot# BRANDING_JEV_SCREENSHOT_PERCENT=0-# optional OpenAI-compatible provider for JSON extraction; routed by team cohort-# at ROLLOUT_PERCENT, mirrored at SHADOW_PERCENT, with per-request fallback-EXTRACTION_PROVIDER_BASE_URL=-EXTRACTION_PROVIDER_API_KEY=-EXTRACTION_PROVIDER_MODEL=extraction-v1-EXTRACTION_PROVIDER_ROLLOUT_PERCENT=0-EXTRACTION_PROVIDER_SHADOW_PERCENT=0-# EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT= # rollout for schemas with $ref/$defs; unset = ROLLOUT_PERCENT-# EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS=true # all-empty-arrays answers on long pages fall back to the default model-EXTRACTION_PROVIDER_TEAM_IDS=-# EXTRACTION_PROVIDER_REQUEST_OPT_IN=false-# inputs above this many bytes stay on the default model-# EXTRACTION_PROVIDER_MAX_INPUT_BYTES=400000-# amortized price of the provider model, per million tokens, for cost tracking-# EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK=0.07-# EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK=0.2BULL_AUTH_KEY=@# optional second bearer value accepted on /admin/ops/* while callers move to a new BULL_AUTH_KEY# BULL_AUTH_KEY_PREVIOUS=diff --git a/apps/api/src/__tests__/snips/v2/extraction-provider.test.ts b/apps/api/src/__tests__/snips/v2/extraction-provider.test.tsdeleted file mode 100644index 7d278f5ad..000000000--- a/apps/api/src/__tests__/snips/v2/extraction-provider.test.ts+++ /dev/null@@ -1,84 +0,0 @@-import {- scrape,- scrapeWithFailure,- scrapeTimeout,- idmux,- Identity,-} from "./lib";--// Runs only where the API under test is guaranteed to route to the provider:-// a configured base URL and either a 100% rollout or the per-request opt-in.-const configured =- !!process.env.EXTRACTION_PROVIDER_BASE_URL &&- (process.env.EXTRACTION_PROVIDER_ROLLOUT_PERCENT === "100" ||- ["true", "1"].includes(- (process.env.EXTRACTION_PROVIDER_REQUEST_OPT_IN ?? "").toLowerCase(),- ));--let identity: Identity;--beforeAll(async () => {- identity = await idmux({- name: "extraction-provider",- concurrency: 10,- credits: 1_000_000,- });-}, 10000);--(configured ? describe : describe.skip)(- "JSON extraction via the extraction provider",- () => {- it.concurrent(- "returns a schema-conforming object",- async () => {- const response = await scrape(- {- url: "https://firecrawl.dev",- formats: [- {- type: "json",- schema: {- type: "object",- properties: {- company_name: { type: "string" },- tagline: { type: "string" },- },- required: ["company_name", "tagline"],- },- },- ],- },- identity,- );- expect(response.json).toBeDefined();- expect(typeof response.json.company_name).toBe("string");- expect(typeof response.json.tagline).toBe("string");- expect(response.json.company_name.toLowerCase()).toContain("firecrawl");- },- scrapeTimeout,- );- },-);--describe("JSON format model option", () => {- it.concurrent(- "rejects a model that is not available",- async () => {- const response = await scrapeWithFailure(- {- url: "https://firecrawl.dev",- formats: [- {- type: "json",- model: "not-a-configured-extraction-model",- schema: { type: "object", properties: { x: { type: "string" } } },- },- ],- },- identity,- );- expect(response.error).toContain("Unsupported extraction model");- },- scrapeTimeout,- );-});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex bd21dfdae..6a2e7fcec 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -717,56 +717,6 @@ const configSchema = z.object({// AI/MLMODEL_NAME: z.string().optional(),- // Optional OpenAI-compatible provider for JSON extraction. Traffic is routed- // by team cohort (stable hash) at ROLLOUT_PERCENT, mirrored without using the- // result at SHADOW_PERCENT, and always routed for TEAM_IDS. Any failure falls- // back to the default model for that request.- EXTRACTION_PROVIDER_BASE_URL: blankStringAsUnset(z.string().url().optional()),- EXTRACTION_PROVIDER_API_KEY: blankStringAsUnset(z.string().optional()),- EXTRACTION_PROVIDER_MODEL: blankStringAsUnset(- z.string().min(1).default("extraction-v1"),- ),- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: z.coerce- .number()- .int()- .min(0)- .max(100)- .default(0),- EXTRACTION_PROVIDER_SHADOW_PERCENT: z.coerce- .number()- .int()- .min(0)- .max(100)- .default(0),- // Rollout percentage for schemas with $ref/$defs (the ones the default- // path sends to its larger model); unset = same as ROLLOUT_PERCENT.- EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: blankStringAsUnset(- z.coerce.number().int().min(0).max(100).optional(),- ),- // A provider answer whose arrays are all empty, for a page long enough to- // plausibly hold items, counts as a failure and falls back to the default- // model for that request.- EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: z- .stringbool()- .optional()- .default(true),- EXTRACTION_PROVIDER_TEAM_IDS: z.string().optional(),- // Let any team pick the provider per request (json format `model`); off =- // only teams in TEAM_IDS may.- EXTRACTION_PROVIDER_REQUEST_OPT_IN: z.stringbool().optional(),- EXTRACTION_PROVIDER_MAX_INPUT_BYTES: z.coerce- .number()- .int()- .nonnegative()- .default(400_000),- EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK: z.coerce- .number()- .nonnegative()- .default(0.07),- EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK: z.coerce- .number()- .nonnegative()- .default(0.2),MODEL_EMBEDDING_NAME: z.string().optional(),OLLAMA_BASE_URL: z.string().optional(),VERTEX_CREDENTIALS: z.string().optional(),@@ -889,22 +839,6 @@ const configSchema = z.object({});const validatedConfigSchema = configSchema.superRefine((value, context) => {- for (const key of [- "EXTRACTION_PROVIDER_ROLLOUT_PERCENT",- "EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT",- ] as const) {- const rollout = value[key];- if (- rollout !== undefined &&- rollout + value.EXTRACTION_PROVIDER_SHADOW_PERCENT > 100- ) {- context.addIssue({- code: "custom",- path: ["EXTRACTION_PROVIDER_SHADOW_PERCENT"],- message: `${key} + EXTRACTION_PROVIDER_SHADOW_PERCENT must not exceed 100`,- });- }- }if (value.GOV_ROUTING_ENABLED && !value.SEARCH_PLATFORM_URL) {context.addIssue({code: "custom",diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex bc15c3b0b..6a387357f 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -3,7 +3,6 @@ import type { ThirdPartyDataTermsRequiredError } from "../../lib/exchange";import { Request, Response } from "express";import { hasCategory } from "../../lib/search-query-builder";import { config } from "../../config";-import { isRequestedModelSupported } from "../../lib/extraction-provider";import { z } from "zod";import { browserProfileNameSchema } from "../../lib/browser-profiles";import { protocolIncluded, checkUrl } from "../../lib/validateUrl";@@ -367,14 +366,6 @@ const jsonFormatWithOptions = z.strictObject({}).superRefine(addStrictSchemaIssue),prompt: z.string().max(10000).optional(),- // Optional: name a configured extraction model for this request.- model: z- .string()- .max(100)- .optional()- .refine(val => val === undefined || isRequestedModelSupported(val), {- message: "Unsupported extraction model.",- }),// Deprecated: lifted into the top-level checkPromptInjection at parse time.checkPromptInjection: z.boolean().optional(),});diff --git a/apps/api/src/lib/extraction-provider.test.ts b/apps/api/src/lib/extraction-provider.test.tsdeleted file mode 100644index 9a9371a6c..000000000--- a/apps/api/src/lib/extraction-provider.test.ts+++ /dev/null@@ -1,278 +0,0 @@-import { config } from "../config";-import {- cohortBucket,- extractionProviderCost,- isEmptyListAnswer,- isRequestedModelSupported,- selectExtractionRoute,-} from "./extraction-provider";--const base = { teamId: "team-a", zeroDataRetention: false, inputBytes: 10_000 };--function withConfig<T>(overrides: Partial<typeof config>, fn: () => T): T {- const saved: Partial<typeof config> = {};- for (const k of Object.keys(overrides) as (keyof typeof config)[]) {- (saved as any)[k] = config[k];- (config as any)[k] = overrides[k];- }- try {- return fn();- } finally {- Object.assign(config, saved);- }-}--describe("extraction provider routing", () => {- it("is off when no provider is configured", () => {- withConfig(- {- EXTRACTION_PROVIDER_BASE_URL: undefined,- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,- },- () => expect(selectExtractionRoute(base)).toBe("default"),- );- });-- it("buckets teams stably and evenly enough to roll out by percentage", () => {- const a = cohortBucket("team-a");- expect(cohortBucket("team-a")).toBe(a);- expect(a).toBeGreaterThanOrEqual(0);- expect(a).toBeLessThan(100);- const buckets = new Set(- Array.from({ length: 2000 }, (_, i) => cohortBucket(`team-${i}`)),- );- expect(buckets.size).toBe(100);- });-- it("routes below the rollout percentage, shadows in the next band, defaults above", () => {- withConfig(- {- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 10,- EXTRACTION_PROVIDER_SHADOW_PERCENT: 10,- EXTRACTION_PROVIDER_TEAM_IDS: undefined,- },- () => {- const teams = Array.from({ length: 500 }, (_, i) => `team-${i}`);- for (const teamId of teams) {- const b = cohortBucket(teamId);- const expected = b < 10 ? "provider" : b < 20 ? "shadow" : "default";- expect(selectExtractionRoute({ ...base, teamId })).toBe(expected);- }- },- );- });-- it("routes 100% and 0% as expected", () => {- withConfig(- {- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,- EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,- },- () => expect(selectExtractionRoute(base)).toBe("provider"),- );- withConfig(- {- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,- EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,- EXTRACTION_PROVIDER_TEAM_IDS: undefined,- },- () => expect(selectExtractionRoute(base)).toBe("default"),- );- });-- it("keeps zero-data-retention requests, oversized inputs and unknown teams on the default model", () => {- withConfig(- {- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,- EXTRACTION_PROVIDER_MAX_INPUT_BYTES: 400_000,- },- () => {- expect(- selectExtractionRoute({ ...base, zeroDataRetention: true }),- ).toBe("default");- expect(selectExtractionRoute({ ...base, inputBytes: 400_001 })).toBe(- "default",- );- expect(selectExtractionRoute({ ...base, teamId: undefined })).toBe(- "default",- );- },- );- });-- it("always routes allowlisted teams", () => {- withConfig(- {- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,- EXTRACTION_PROVIDER_TEAM_IDS: "x, team-a ,y",- },- () => {- expect(selectExtractionRoute(base)).toBe("provider");- expect(selectExtractionRoute({ ...base, teamId: "team-b" })).toBe(- "default",- );- },- );- });-- it("prices only the configured provider model", () => {- withConfig(- {- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",- EXTRACTION_PROVIDER_MODEL: "extraction-v1",- EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK: 0.07,- EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK: 0.2,- },- () => {- expect(extractionProviderCost("extraction-v1")).toEqual({- input_cost: 0.07,- output_cost: 0.2,- });- expect(extractionProviderCost("gpt-4o-mini")).toBeUndefined();- },- );- });-- it("honours a requested model for listed teams, or for everyone with the opt-in switch", () => {- const cfg = {- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",- EXTRACTION_PROVIDER_MODEL: "extraction-v1",- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,- EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,- };- withConfig(- {- ...cfg,- EXTRACTION_PROVIDER_TEAM_IDS: "team-a",- EXTRACTION_PROVIDER_REQUEST_OPT_IN: false,- },- () => {- expect(- selectExtractionRoute({ ...base, requestedModel: "extraction-v1" }),- ).toBe("provider");- expect(- selectExtractionRoute({- ...base,- teamId: "team-b",- requestedModel: "extraction-v1",- }),- ).toBe("default");- expect(- selectExtractionRoute({ ...base, requestedModel: "something-else" }),- ).toBe("default");- expect(- selectExtractionRoute({- ...base,- requestedModel: "extraction-v1",- zeroDataRetention: true,- }),- ).toBe("default");- },- );- withConfig(- {- ...cfg,- EXTRACTION_PROVIDER_TEAM_IDS: undefined,- EXTRACTION_PROVIDER_REQUEST_OPT_IN: true,- },- () => {- expect(- selectExtractionRoute({- ...base,- teamId: "team-b",- requestedModel: "extraction-v1",- }),- ).toBe("provider");- },- );- withConfig({ ...cfg }, () => {- expect(isRequestedModelSupported("extraction-v1")).toBe(true);- expect(isRequestedModelSupported("gpt-4o-mini")).toBe(false);- });- withConfig({ ...cfg, EXTRACTION_PROVIDER_BASE_URL: undefined }, () =>- expect(isRequestedModelSupported("extraction-v1")).toBe(false),- );- });-});--describe("ref-schema rollout", () => {- const provider = {- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",- EXTRACTION_PROVIDER_TEAM_IDS: undefined,- EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,- };-- it("uses REF_SCHEMA_ROLLOUT_PERCENT for ref schemas and ROLLOUT_PERCENT otherwise", () => {- withConfig(- {- ...provider,- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,- EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: 100,- },- () => {- expect(selectExtractionRoute({ ...base, refSchema: true })).toBe(- "provider",- );- expect(selectExtractionRoute({ ...base, refSchema: false })).toBe(- "default",- );- expect(selectExtractionRoute(base)).toBe("default");- },- );- });-- it("falls back to ROLLOUT_PERCENT when the ref percentage is unset", () => {- withConfig(- {- ...provider,- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,- EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: undefined,- },- () =>- expect(selectExtractionRoute({ ...base, refSchema: true })).toBe(- "provider",- ),- );- });-- it("keeps the shadow band directly above the ref rollout band", () => {- withConfig(- {- ...provider,- EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,- EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: 20,- EXTRACTION_PROVIDER_SHADOW_PERCENT: 10,- },- () => {- for (let i = 0; i < 300; i++) {- const teamId = `team-${i}`;- const b = cohortBucket(teamId);- expect(- selectExtractionRoute({ ...base, teamId, refSchema: true }),- ).toBe(b < 20 ? "provider" : b < 30 ? "shadow" : "default");- expect(selectExtractionRoute({ ...base, teamId })).toBe(- b < 10 ? "shadow" : "default",- );- }- },- );- });-});--describe("isEmptyListAnswer", () => {- it("is true only when the answer has arrays and all of them are empty", () => {- expect(isEmptyListAnswer({ jobs: [] })).toBe(true);- expect(isEmptyListAnswer({ page: { items: [], tags: [] } })).toBe(true);- expect(isEmptyListAnswer([])).toBe(true);- expect(isEmptyListAnswer({ jobs: [{ title: "Cook" }] })).toBe(false);- expect(isEmptyListAnswer({ jobs: [], seen: [1] })).toBe(false);- expect(isEmptyListAnswer({ title: "No lists here" })).toBe(false);- expect(isEmptyListAnswer(undefined)).toBe(false);- expect(isEmptyListAnswer(null)).toBe(false);- });-});diff --git a/apps/api/src/lib/extraction-provider.ts b/apps/api/src/lib/extraction-provider.tsdeleted file mode 100644index 971f54512..000000000--- a/apps/api/src/lib/extraction-provider.ts+++ /dev/null@@ -1,119 +0,0 @@-import { createHash } from "crypto";-import { config } from "../config";--/**- * Routing for the optional JSON extraction provider (EXTRACTION_PROVIDER_*).- *- * A team lands in a stable cohort bucket 0-99 (sha1 of the team id), so a- * rollout percentage selects the same teams every time and can be widened- * without reshuffling. Buckets below ROLLOUT_PERCENT are routed; the next- * SHADOW_PERCENT buckets are mirrored (the provider is called, the result is- * discarded). Teams listed in TEAM_IDS are always routed. Zero-data-retention- * requests and inputs over MAX_INPUT_BYTES stay on the default model.- *- * A request may also name the provider model explicitly (json format `model`);- * that routes it regardless of cohort when the team is listed in TEAM_IDS or- * REQUEST_OPT_IN is on. Unknown model names are rejected at validation.- *- * Schemas with $ref/$defs can roll out on their own percentage- * (REF_SCHEMA_ROLLOUT_PERCENT), since the default path prices them- * differently. An answer that is nothing but empty arrays on a page long- * enough to hold items is treated as a provider failure (see- * isEmptyListAnswer) so the request falls back instead of returning nothing.- */-type ExtractionRoute = "provider" | "shadow" | "default";--function isExtractionProviderConfigured(): boolean {- return !!config.EXTRACTION_PROVIDER_BASE_URL;-}--export function cohortBucket(teamId: string): number {- const digest = createHash("sha1").update(teamId).digest();- return digest.readUInt32BE(0) % 100;-}--function allowlistedTeams(): Set<string> {- return new Set(- (config.EXTRACTION_PROVIDER_TEAM_IDS ?? "")- .split(",")- .map(s => s.trim())- .filter(Boolean),- );-}--/** Model names a request may ask for in the json format `model` field. */-export function isRequestedModelSupported(model: string): boolean {- return (- isExtractionProviderConfigured() &&- model === config.EXTRACTION_PROVIDER_MODEL- );-}--export function selectExtractionRoute(input: {- teamId: string | undefined;- zeroDataRetention: boolean;- inputBytes: number;- requestedModel?: string;- /** The schema uses $ref/$defs (routed by REF_SCHEMA_ROLLOUT_PERCENT when set). */- refSchema?: boolean;-}): ExtractionRoute {- if (!isExtractionProviderConfigured()) return "default";- if (input.zeroDataRetention) return "default";- if (input.inputBytes > config.EXTRACTION_PROVIDER_MAX_INPUT_BYTES)- return "default";- if (!input.teamId) return "default";- const listed = allowlistedTeams().has(input.teamId);- if (input.requestedModel !== undefined) {- return isRequestedModelSupported(input.requestedModel) &&- (listed || config.EXTRACTION_PROVIDER_REQUEST_OPT_IN)- ? "provider"- : "default";- }- if (listed) return "provider";- const bucket = cohortBucket(input.teamId);- const rollout =- input.refSchema &&- config.EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT !== undefined- ? config.EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT- : config.EXTRACTION_PROVIDER_ROLLOUT_PERCENT;- if (bucket < rollout) return "provider";- if (bucket < rollout + config.EXTRACTION_PROVIDER_SHADOW_PERCENT)- return "shadow";- return "default";-}--/** Pages shorter than this can legitimately have no list items. */-export const EMPTY_LIST_FALLBACK_MIN_CHARS = 2000;--function arraysIn(value: unknown, out: unknown[][] = []): unknown[][] {- if (Array.isArray(value)) {- out.push(value);- for (const v of value) arraysIn(v, out);- } else if (value && typeof value === "object") {- for (const v of Object.values(value as Record<string, unknown>))- arraysIn(v, out);- }- return out;-}--/**- * True when the answer contains arrays and every one of them is empty — the- * shape a model produces when it gives up on a listing page. Only meaningful- * with a markdown input of at least EMPTY_LIST_FALLBACK_MIN_CHARS.- */-export function isEmptyListAnswer(answer: unknown): boolean {- const arrays = arraysIn(answer);- return arrays.length > 0 && arrays.every(a => a.length === 0);-}--/** Per-million-token prices for the provider model (amortized infrastructure cost, configured). */-export function extractionProviderCost(- model: string,-): { input_cost: number; output_cost: number } | undefined {- if (!isExtractionProviderConfigured()) return undefined;- if (model !== config.EXTRACTION_PROVIDER_MODEL) return undefined;- return {- input_cost: config.EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK,- output_cost: config.EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK,- };-}diff --git a/apps/api/src/lib/generic-ai.ts b/apps/api/src/lib/generic-ai.tsindex 24f6922cf..26c9d9e6c 100644--- a/apps/api/src/lib/generic-ai.ts+++ b/apps/api/src/lib/generic-ai.ts@@ -23,11 +23,8 @@ type Provider =| "openrouter"| "fireworks"| "deepinfra"- | "vertex"- | "extraction";-const defaultProvider: Exclude<Provider, "extraction"> = config.OLLAMA_BASE_URL- ? "ollama"- : "openai";+ | "vertex";+const defaultProvider: Provider = config.OLLAMA_BASE_URL ? "ollama" : "openai";const vertexAuthOptions = config.VERTEX_CREDENTIALS? {@@ -53,12 +50,6 @@ const providerList: Record<Provider, any> = {}),fireworks, //FIREWORKS_API_KEYdeepinfra, //DEEPINFRA_API_KEY- // OpenAI-compatible JSON extraction provider (EXTRACTION_PROVIDER_*); Chat- // Completions only, see getModel.- extraction: createOpenAI({- apiKey: config.EXTRACTION_PROVIDER_API_KEY ?? "unused",- baseURL: config.EXTRACTION_PROVIDER_BASE_URL,- }),vertex: createVertex({project: "firecrawl",//https://github.com/vercel/ai/issues/6644 bug@@ -164,17 +155,12 @@ export function getModel(if (provider === "openai" && modelName.startsWith("o3-mini")) {return withUsageTelemetry(providerList.openai.chat(modelName));}- // The extraction provider speaks Chat Completions, not the Responses API,- // and is not subject to the MODEL_NAME override.- if (provider === "extraction") {- return withUsageTelemetry(providerList.extraction.chat(name));- }return withUsageTelemetry(providerList[provider](modelName));}export function getEmbeddingModel(name: string,- provider: Exclude<Provider, "extraction"> = defaultProvider,+ provider: Provider = defaultProvider,options?: Pick<ModelOptions, "teamFlags" | "feature">,) {const modelName = config.MODEL_EMBEDDING_NAME || name;diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.tsindex 4d02ab07a..25950002b 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts@@ -19,7 +19,6 @@ import { performSummary } from "./llmExtract";import { performCleanContent } from "./llmExtract";import { encoding_for_model } from "@dqbd/tiktoken";import type { Mock } from "vitest";-import { config } from "../../../config";vi.mock("@dqbd/tiktoken", () => ({encoding_for_model: vi.fn(),@@ -483,127 +482,3 @@ describe("performLLMExtract empty-result handling", () => {);});});--describe("performLLMExtract extraction provider fallback", () => {- const listSchema = {- type: "object",- properties: {- jobs: {- type: "array",- items: { type: "object", properties: { title: { type: "string" } } },- },- },- };- const longPage = "Open roles: Line cook. Pizza cook. Apply now. ".repeat(60); // > 2,000 chars- const saved: Partial<typeof config> = {};-- function withProvider(overrides: Partial<typeof config> = {}) {- const values: Partial<typeof config> = {- EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",- EXTRACTION_PROVIDER_MODEL: "extraction-v1",- EXTRACTION_PROVIDER_TEAM_IDS: "test-team",- EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: true,- ...overrides,- };- for (const k of Object.keys(values) as (keyof typeof config)[]) {- (saved as any)[k] = config[k];- (config as any)[k] = values[k];- }- }-- function meta() {- const m = makeJsonMeta();- m.options.formats[0].schema = listSchema as any;- (m.logger as any).warn = vi.fn(); // the fallback path logs a warning- return m;- }-- function modelIds() {- return (extractData as Mock).mock.calls.map(- c => c[0].extractOptions.model.modelId,- );- }-- beforeEach(() => {- (extractData as Mock).mockReset();- });-- afterEach(() => {- Object.assign(config, saved);- for (const k of Object.keys(saved)) delete (saved as any)[k];- });-- it("falls back to the default model when the provider answers with only empty lists on a long page", async () => {- withProvider();- (extractData as Mock)- .mockResolvedValueOnce({- extractedDataArray: [{ jobs: [] }],- warning: undefined,- costLimitExceededTokenUsage: null,- })- .mockResolvedValueOnce({- extractedDataArray: [{ jobs: [{ title: "Line cook" }] }],- warning: undefined,- costLimitExceededTokenUsage: null,- });-- const result = await performLLMExtract(meta(), {- markdown: longPage,- metadata: {},- } as any);-- expect(modelIds()).toEqual(["extraction-v1", "gpt-4o-mini"]);- expect(result.json).toEqual({ jobs: [{ title: "Line cook" }] });- });-- it("keeps the provider's empty lists on a short page", async () => {- withProvider();- (extractData as Mock).mockResolvedValueOnce({- extractedDataArray: [{ jobs: [] }],- warning: undefined,- costLimitExceededTokenUsage: null,- });-- const result = await performLLMExtract(meta(), {- markdown: "No openings right now.",- metadata: {},- } as any);-- expect(modelIds()).toEqual(["extraction-v1"]);- expect(result.json).toEqual({ jobs: [] });- });-- it("keeps the provider's empty lists when the fallback is switched off", async () => {- withProvider({ EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: false });- (extractData as Mock).mockResolvedValueOnce({- extractedDataArray: [{ jobs: [] }],- warning: undefined,- costLimitExceededTokenUsage: null,- });-- const result = await performLLMExtract(meta(), {- markdown: longPage,- metadata: {},- } as any);-- expect(modelIds()).toEqual(["extraction-v1"]);- expect(result.json).toEqual({ jobs: [] });- });-- it("does not fall back when the provider found items", async () => {- withProvider();- (extractData as Mock).mockResolvedValueOnce({- extractedDataArray: [{ jobs: [{ title: "Pizza cook" }] }],- warning: undefined,- costLimitExceededTokenUsage: null,- });-- const result = await performLLMExtract(meta(), {- markdown: longPage,- metadata: {},- } as any);-- expect(modelIds()).toEqual(["extraction-v1"]);- expect(result.json).toEqual({ jobs: [{ title: "Pizza cook" }] });- });-});diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 8fe6491fa..bea6849fa 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -32,13 +32,6 @@ import { CostTracking } from "../../../lib/cost-tracking";import { isZeroDataRetentionActive } from "../../../lib/otel-tracer";import { isAgentExtractModelValid } from "../../../controllers/v1/types";import { hasFormatOfType } from "../../../lib/format-utils";-import {- EMPTY_LIST_FALLBACK_MIN_CHARS,- extractionProviderCost,- isEmptyListAnswer,- selectExtractionRoute,-} from "../../../lib/extraction-provider";-import { config } from "../../../config";import { isUsOnly } from "../../../lib/us-only";// Smart model selection based on schema@@ -316,10 +309,6 @@ function lookupModelCost(const slash = model.indexOf("/");if (slash !== -1) candidates.push(model.slice(slash + 1));- // The configured extraction provider is priced from config even if its- // name collides with a static entry.- const providerCost = extractionProviderCost(model);- if (providerCost) return providerCost;for (const id of candidates) {if (modelCosts[id]) return modelCosts[id];}@@ -393,8 +382,6 @@ export type GenerateCompletionsOptions = {isExtractEndpoint?: boolean;mode?: "object" | "no-object";providerOptions?: LanguageModelV1ProviderMetadata;- /** AI SDK retry count for the primary model call (default: the SDK's). */- maxRetries?: number;retryModel?: LanguageModel;costTrackingOptions: {costTracking: CostTracking;@@ -464,7 +451,6 @@ export async function generateCompletions({model = getModel("gpt-4o-mini", "openai", { teamFlags }),mode = "object",providerOptions,- maxRetries,retryModel = getModel("gpt-4.1-mini", "openai", { teamFlags }),costTrackingOptions,metadata,@@ -794,7 +780,6 @@ export async function generateCompletions({const generateObjectConfig = {model: currentModel,prompt: prompt,- ...(maxRetries !== undefined && { maxRetries }),providerOptions: {...(providerOptions || {}),google: {@@ -902,7 +887,6 @@ export async function generateCompletions({const retryConfig = {...generateObjectConfig,model: currentModel,- maxRetries: undefined, // the fallback model keeps the SDK default};result = await generateObject(retryConfig);costTrackingOptions.costTracking.addCall({@@ -1094,29 +1078,9 @@ export async function performLLMExtract(teamFlags,};- // Optional extraction provider: routed by team cohort, with the default- // model as a per-request fallback; or mirrored (shadow) without using- // the result. The agent path and US-only teams are never routed.- const route =- useAgent || usOnly- ? "default"- : selectExtractionRoute({- teamId: meta.internalOptions.teamId,- zeroDataRetention:- !!meta.internalOptions.zeroDataRetention ||- isZeroDataRetentionActive(),- inputBytes: Buffer.byteLength(document.markdown ?? "", "utf8"),- requestedModel: jsonFormat.model,- refSchema: modelSelection.reason === "recursive_schema_detected",- });- if (jsonFormat.model !== undefined && route !== "provider") {- document.warning =- "The requested extraction model is not available for this request; the default model was used." +- (document.warning ? " " + document.warning : "");- }- const extractWith = (options: GenerateCompletionsOptions) =>- extractData({- extractOptions: options,+ const { extractedDataArray, warning, costLimitExceededTokenUsage } =+ await extractData({+ extractOptions: generationOptions,urls: [meta.rewrittenUrl ?? meta.url],useAgent,checkPromptInjection: meta.options.checkPromptInjection,@@ -1126,74 +1090,6 @@ export async function performLLMExtract(functionId: llmFunctionId,},});- const providerOptions = (): GenerateCompletionsOptions => ({- ...generationOptions,- model: getModel(config.EXTRACTION_PROVIDER_MODEL, "extraction"),- retryModel: getModel(modelSelection.modelName, "openai"),- // An unreachable provider should fall back at once, not after the- // SDK's retries (~6 s of connection attempts per request).- maxRetries: 0,- });-- // extractData reports generation failures as a warning rather than- // throwing, so both paths count as a provider failure.- const PROVIDER_FAILURE = "JSON extraction failed:";- let extraction: Awaited<ReturnType<typeof extractWith>>;- if (route === "provider") {- let attempt: Awaited<ReturnType<typeof extractWith>> | undefined;- let failure: string | undefined;- try {- attempt = await extractWith(providerOptions());- if (attempt.warning?.includes(PROVIDER_FAILURE)) {- failure = attempt.warning;- } else if (- config.EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS &&- (document.markdown?.length ?? 0) >= EMPTY_LIST_FALLBACK_MIN_CHARS &&- isEmptyListAnswer(- attempt.extractedDataArray[attempt.extractedDataArray.length - 1],- )- ) {- failure = "provider returned only empty lists";- }- } catch (error) {- failure = (error as Error).message;- }- if (attempt !== undefined && failure === undefined) {- extraction = attempt;- } else {- meta.logger.warn(- "Extraction provider failed, falling back to the default model",- { error: failure },- );- extraction = await extractWith(generationOptions);- }- } else {- extraction = await extractWith(generationOptions);- if (route === "shadow") {- // Fire and forget: the provider's own telemetry records the outcome;- // nothing is billed to the team and nothing changes in the response.- void extractWith({- ...providerOptions(),- costTrackingOptions: {- costTracking: new CostTracking(),- metadata: {- module: "scrapeURL",- method: "performLLMExtract/shadow",- },- },- metadata: {- ...generationOptions.metadata,- functionId: "performLLMExtract/shadow",- },- }).catch(error => {- meta.logger.debug("Extraction provider shadow call failed", {- error: (error as Error).message,- });- });- }- }- const { extractedDataArray, warning, costLimitExceededTokenUsage } =- extraction;if (warning) {document.warning =