Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): extraction provider — separate rollout for ref schemas, fall back on empty-list answers (#5559)
Abimael Martell4e327a6170e9
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex d2d99f1ac..9e5af838a 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -84,6 +84,8 @@ EXTRACTION_PROVIDER_API_KEY=EXTRACTION_PROVIDER_MODEL=extraction-v1EXTRACTION_PROVIDER_ROLLOUT_PERCENT=0EXTRACTION_PROVIDER_SHADOW_PERCENT=0+# EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT= # rollout for schemas with $ref/$defs; unset = ROLLOUT_PERCENT+# EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS=true # all-empty-arrays answers on long pages fall back to the default modelEXTRACTION_PROVIDER_TEAM_IDS=# EXTRACTION_PROVIDER_REQUEST_OPT_IN=false# inputs above this many bytes stay on the default modeldiff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex e96abc080..242c5e6af 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -713,6 +713,18 @@ const configSchema = z.object({.min(0).max(100).default(0),+ // Rollout percentage for schemas with $ref/$defs (the ones the default+ // path sends to its larger model); unset = same as ROLLOUT_PERCENT.+ EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: blankStringAsUnset(+ z.coerce.number().int().min(0).max(100).optional(),+ ),+ // A provider answer whose arrays are all empty, for a page long enough to+ // plausibly hold items, counts as a failure and falls back to the default+ // model for that request.+ EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: z+ .stringbool()+ .optional()+ .default(true),EXTRACTION_PROVIDER_TEAM_IDS: z.string().optional(),// Let any team pick the provider per request (json format `model`); off =// only teams in TEAM_IDS may.@@ -852,17 +864,21 @@ const configSchema = z.object({});const validatedConfigSchema = configSchema.superRefine((value, context) => {- if (- value.EXTRACTION_PROVIDER_ROLLOUT_PERCENT +- value.EXTRACTION_PROVIDER_SHADOW_PERCENT >- 100- ) {- context.addIssue({- code: "custom",- path: ["EXTRACTION_PROVIDER_SHADOW_PERCENT"],- message:- "EXTRACTION_PROVIDER_ROLLOUT_PERCENT + EXTRACTION_PROVIDER_SHADOW_PERCENT must not exceed 100",- });+ for (const key of [+ "EXTRACTION_PROVIDER_ROLLOUT_PERCENT",+ "EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT",+ ] as const) {+ const rollout = value[key];+ if (+ rollout !== undefined &&+ rollout + value.EXTRACTION_PROVIDER_SHADOW_PERCENT > 100+ ) {+ context.addIssue({+ code: "custom",+ path: ["EXTRACTION_PROVIDER_SHADOW_PERCENT"],+ message: `${key} + EXTRACTION_PROVIDER_SHADOW_PERCENT must not exceed 100`,+ });+ }}for (const error of getMcpActionLogConfigErrors(value)) {context.addIssue({diff --git a/apps/api/src/lib/branding/font-names.ts b/apps/api/src/lib/branding/font-names.tsindex 18664fb5a..5e0925263 100644--- a/apps/api/src/lib/branding/font-names.ts+++ b/apps/api/src/lib/branding/font-names.ts@@ -39,19 +39,16 @@ const SYSTEM_STACK_HEADS = new Set([* real name further down the stack: Wix's `wfont_<site>_<hash>` and* `wf_<hash>`, then `orig_<family>`. The alias says nothing about the font.*/-export const isHashedAlias = (family: string): boolean =>+const isHashedAlias = (family: string): boolean =>/^wfont_[0-9a-f]{6}_[0-9a-f]{32}$/i.test(family) ||/^wf_[0-9a-f]{20,}$/i.test(family) ||/^[a-z]{1,6}_[0-9a-f]{24,}$/i.test(family);const unquote = (family: string) =>- family.trim().replace(/^["']|["']$/g, "").trim();--/** A stack that asks for the visitor's system font. */-export const isSystemStack = (stack: readonly string[]): boolean => {- const first = stack.map(unquote).find(Boolean);- return !!first && SYSTEM_STACK_HEADS.has(first.toLowerCase());-};+ family+ .trim()+ .replace(/^["']|["']$/g, "")+ .trim();/*** What a stack is set in when it names no brand face: the system font@@ -135,7 +132,10 @@ export const titleCase = (name: string): string =>* floor the processor's own output also meets.*/export function displayFamily(name: string): string {- let out = normalizeVendorName(name).replace(/_/g, " ").replace(/\s+/g, " ").trim();+ let out = normalizeVendorName(name)+ .replace(/_/g, " ")+ .replace(/\s+/g, " ")+ .trim();if (out && out === out.toLowerCase()) out = titleCase(out);return knownFamily(out) ?? out;}diff --git a/apps/api/src/lib/extraction-provider.test.ts b/apps/api/src/lib/extraction-provider.test.tsindex 1a4dfd473..9a9371a6c 100644--- a/apps/api/src/lib/extraction-provider.test.ts+++ b/apps/api/src/lib/extraction-provider.test.ts@@ -2,6 +2,7 @@ import { config } from "../config";import {cohortBucket,extractionProviderCost,+ isEmptyListAnswer,isRequestedModelSupported,selectExtractionRoute,} from "./extraction-provider";@@ -198,3 +199,80 @@ describe("extraction provider routing", () => {);});});++describe("ref-schema rollout", () => {+ const provider = {+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",+ EXTRACTION_PROVIDER_TEAM_IDS: undefined,+ EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,+ };++ it("uses REF_SCHEMA_ROLLOUT_PERCENT for ref schemas and ROLLOUT_PERCENT otherwise", () => {+ withConfig(+ {+ ...provider,+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,+ EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: 100,+ },+ () => {+ expect(selectExtractionRoute({ ...base, refSchema: true })).toBe(+ "provider",+ );+ expect(selectExtractionRoute({ ...base, refSchema: false })).toBe(+ "default",+ );+ expect(selectExtractionRoute(base)).toBe("default");+ },+ );+ });++ it("falls back to ROLLOUT_PERCENT when the ref percentage is unset", () => {+ withConfig(+ {+ ...provider,+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,+ EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: undefined,+ },+ () =>+ expect(selectExtractionRoute({ ...base, refSchema: true })).toBe(+ "provider",+ ),+ );+ });++ it("keeps the shadow band directly above the ref rollout band", () => {+ withConfig(+ {+ ...provider,+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,+ EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: 20,+ EXTRACTION_PROVIDER_SHADOW_PERCENT: 10,+ },+ () => {+ for (let i = 0; i < 300; i++) {+ const teamId = `team-${i}`;+ const b = cohortBucket(teamId);+ expect(+ selectExtractionRoute({ ...base, teamId, refSchema: true }),+ ).toBe(b < 20 ? "provider" : b < 30 ? "shadow" : "default");+ expect(selectExtractionRoute({ ...base, teamId })).toBe(+ b < 10 ? "shadow" : "default",+ );+ }+ },+ );+ });+});++describe("isEmptyListAnswer", () => {+ it("is true only when the answer has arrays and all of them are empty", () => {+ expect(isEmptyListAnswer({ jobs: [] })).toBe(true);+ expect(isEmptyListAnswer({ page: { items: [], tags: [] } })).toBe(true);+ expect(isEmptyListAnswer([])).toBe(true);+ expect(isEmptyListAnswer({ jobs: [{ title: "Cook" }] })).toBe(false);+ expect(isEmptyListAnswer({ jobs: [], seen: [1] })).toBe(false);+ expect(isEmptyListAnswer({ title: "No lists here" })).toBe(false);+ expect(isEmptyListAnswer(undefined)).toBe(false);+ expect(isEmptyListAnswer(null)).toBe(false);+ });+});diff --git a/apps/api/src/lib/extraction-provider.ts b/apps/api/src/lib/extraction-provider.tsindex 32ee13c93..971f54512 100644--- a/apps/api/src/lib/extraction-provider.ts+++ b/apps/api/src/lib/extraction-provider.ts@@ -14,6 +14,12 @@ import { config } from "../config";* A request may also name the provider model explicitly (json format `model`);* that routes it regardless of cohort when the team is listed in TEAM_IDS or* REQUEST_OPT_IN is on. Unknown model names are rejected at validation.+ *+ * Schemas with $ref/$defs can roll out on their own percentage+ * (REF_SCHEMA_ROLLOUT_PERCENT), since the default path prices them+ * differently. An answer that is nothing but empty arrays on a page long+ * enough to hold items is treated as a provider failure (see+ * isEmptyListAnswer) so the request falls back instead of returning nothing.*/type ExtractionRoute = "provider" | "shadow" | "default";@@ -48,6 +54,8 @@ export function selectExtractionRoute(input: {zeroDataRetention: boolean;inputBytes: number;requestedModel?: string;+ /** The schema uses $ref/$defs (routed by REF_SCHEMA_ROLLOUT_PERCENT when set). */+ refSchema?: boolean;}): ExtractionRoute {if (!isExtractionProviderConfigured()) return "default";if (input.zeroDataRetention) return "default";@@ -63,16 +71,41 @@ export function selectExtractionRoute(input: {}if (listed) return "provider";const bucket = cohortBucket(input.teamId);- if (bucket < config.EXTRACTION_PROVIDER_ROLLOUT_PERCENT) return "provider";- if (- bucket <- config.EXTRACTION_PROVIDER_ROLLOUT_PERCENT +- config.EXTRACTION_PROVIDER_SHADOW_PERCENT- )+ const rollout =+ input.refSchema &&+ config.EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT !== undefined+ ? config.EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT+ : config.EXTRACTION_PROVIDER_ROLLOUT_PERCENT;+ if (bucket < rollout) return "provider";+ if (bucket < rollout + config.EXTRACTION_PROVIDER_SHADOW_PERCENT)return "shadow";return "default";}+/** Pages shorter than this can legitimately have no list items. */+export const EMPTY_LIST_FALLBACK_MIN_CHARS = 2000;++function arraysIn(value: unknown, out: unknown[][] = []): unknown[][] {+ if (Array.isArray(value)) {+ out.push(value);+ for (const v of value) arraysIn(v, out);+ } else if (value && typeof value === "object") {+ for (const v of Object.values(value as Record<string, unknown>))+ arraysIn(v, out);+ }+ return out;+}++/**+ * True when the answer contains arrays and every one of them is empty — the+ * shape a model produces when it gives up on a listing page. Only meaningful+ * with a markdown input of at least EMPTY_LIST_FALLBACK_MIN_CHARS.+ */+export function isEmptyListAnswer(answer: unknown): boolean {+ const arrays = arraysIn(answer);+ return arrays.length > 0 && arrays.every(a => a.length === 0);+}+/** Per-million-token prices for the provider model (amortized infrastructure cost, configured). */export function extractionProviderCost(model: string,diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.tsindex 25950002b..4d02ab07a 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts@@ -19,6 +19,7 @@ import { performSummary } from "./llmExtract";import { performCleanContent } from "./llmExtract";import { encoding_for_model } from "@dqbd/tiktoken";import type { Mock } from "vitest";+import { config } from "../../../config";vi.mock("@dqbd/tiktoken", () => ({encoding_for_model: vi.fn(),@@ -482,3 +483,127 @@ describe("performLLMExtract empty-result handling", () => {);});});++describe("performLLMExtract extraction provider fallback", () => {+ const listSchema = {+ type: "object",+ properties: {+ jobs: {+ type: "array",+ items: { type: "object", properties: { title: { type: "string" } } },+ },+ },+ };+ const longPage = "Open roles: Line cook. Pizza cook. Apply now. ".repeat(60); // > 2,000 chars+ const saved: Partial<typeof config> = {};++ function withProvider(overrides: Partial<typeof config> = {}) {+ const values: Partial<typeof config> = {+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",+ EXTRACTION_PROVIDER_MODEL: "extraction-v1",+ EXTRACTION_PROVIDER_TEAM_IDS: "test-team",+ EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: true,+ ...overrides,+ };+ for (const k of Object.keys(values) as (keyof typeof config)[]) {+ (saved as any)[k] = config[k];+ (config as any)[k] = values[k];+ }+ }++ function meta() {+ const m = makeJsonMeta();+ m.options.formats[0].schema = listSchema as any;+ (m.logger as any).warn = vi.fn(); // the fallback path logs a warning+ return m;+ }++ function modelIds() {+ return (extractData as Mock).mock.calls.map(+ c => c[0].extractOptions.model.modelId,+ );+ }++ beforeEach(() => {+ (extractData as Mock).mockReset();+ });++ afterEach(() => {+ Object.assign(config, saved);+ for (const k of Object.keys(saved)) delete (saved as any)[k];+ });++ it("falls back to the default model when the provider answers with only empty lists on a long page", async () => {+ withProvider();+ (extractData as Mock)+ .mockResolvedValueOnce({+ extractedDataArray: [{ jobs: [] }],+ warning: undefined,+ costLimitExceededTokenUsage: null,+ })+ .mockResolvedValueOnce({+ extractedDataArray: [{ jobs: [{ title: "Line cook" }] }],+ warning: undefined,+ costLimitExceededTokenUsage: null,+ });++ const result = await performLLMExtract(meta(), {+ markdown: longPage,+ metadata: {},+ } as any);++ expect(modelIds()).toEqual(["extraction-v1", "gpt-4o-mini"]);+ expect(result.json).toEqual({ jobs: [{ title: "Line cook" }] });+ });++ it("keeps the provider's empty lists on a short page", async () => {+ withProvider();+ (extractData as Mock).mockResolvedValueOnce({+ extractedDataArray: [{ jobs: [] }],+ warning: undefined,+ costLimitExceededTokenUsage: null,+ });++ const result = await performLLMExtract(meta(), {+ markdown: "No openings right now.",+ metadata: {},+ } as any);++ expect(modelIds()).toEqual(["extraction-v1"]);+ expect(result.json).toEqual({ jobs: [] });+ });++ it("keeps the provider's empty lists when the fallback is switched off", async () => {+ withProvider({ EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: false });+ (extractData as Mock).mockResolvedValueOnce({+ extractedDataArray: [{ jobs: [] }],+ warning: undefined,+ costLimitExceededTokenUsage: null,+ });++ const result = await performLLMExtract(meta(), {+ markdown: longPage,+ metadata: {},+ } as any);++ expect(modelIds()).toEqual(["extraction-v1"]);+ expect(result.json).toEqual({ jobs: [] });+ });++ it("does not fall back when the provider found items", async () => {+ withProvider();+ (extractData as Mock).mockResolvedValueOnce({+ extractedDataArray: [{ jobs: [{ title: "Pizza cook" }] }],+ warning: undefined,+ costLimitExceededTokenUsage: null,+ });++ const result = await performLLMExtract(meta(), {+ markdown: longPage,+ metadata: {},+ } as any);++ expect(modelIds()).toEqual(["extraction-v1"]);+ expect(result.json).toEqual({ jobs: [{ title: "Pizza cook" }] });+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex b253c3b63..8fe6491fa 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -33,7 +33,9 @@ import { isZeroDataRetentionActive } from "../../../lib/otel-tracer";import { isAgentExtractModelValid } from "../../../controllers/v1/types";import { hasFormatOfType } from "../../../lib/format-utils";import {+ EMPTY_LIST_FALLBACK_MIN_CHARS,extractionProviderCost,+ isEmptyListAnswer,selectExtractionRoute,} from "../../../lib/extraction-provider";import { config } from "../../../config";@@ -1105,6 +1107,7 @@ export async function performLLMExtract(isZeroDataRetentionActive(),inputBytes: Buffer.byteLength(document.markdown ?? "", "utf8"),requestedModel: jsonFormat.model,+ refSchema: modelSelection.reason === "recursive_schema_detected",});if (jsonFormat.model !== undefined && route !== "provider") {document.warning =@@ -1143,6 +1146,14 @@ export async function performLLMExtract(attempt = await extractWith(providerOptions());if (attempt.warning?.includes(PROVIDER_FAILURE)) {failure = attempt.warning;+ } else if (+ config.EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS &&+ (document.markdown?.length ?? 0) >= EMPTY_LIST_FALLBACK_MIN_CHARS &&+ isEmptyListAnswer(+ attempt.extractedDataArray[attempt.extractedDataArray.length - 1],+ )+ ) {+ failure = "provider returned only empty lists";}} catch (error) {failure = (error as Error).message;