Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): extraction provider — separate rollout for ref schemas, fall back on empty-list answers (#5559)

Abimael Martell4e327a6170e9
diff --git a/apps/api/.env.example b/apps/api/.env.example
index d2d99f1ac..9e5af838a 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -84,6 +84,8 @@ EXTRACTION_PROVIDER_API_KEY=
EXTRACTION_PROVIDER_MODEL=extraction-v1
EXTRACTION_PROVIDER_ROLLOUT_PERCENT=0
EXTRACTION_PROVIDER_SHADOW_PERCENT=0
+# EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT= # rollout for schemas with $ref/$defs; unset = ROLLOUT_PERCENT
+# EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS=true # all-empty-arrays answers on long pages fall back to the default model
EXTRACTION_PROVIDER_TEAM_IDS=
# EXTRACTION_PROVIDER_REQUEST_OPT_IN=false
# inputs above this many bytes stay on the default model
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index e96abc080..242c5e6af 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -713,6 +713,18 @@ const configSchema = z.object({
.min(0)
.max(100)
.default(0),
+ // Rollout percentage for schemas with $ref/$defs (the ones the default
+ // path sends to its larger model); unset = same as ROLLOUT_PERCENT.
+ EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: blankStringAsUnset(
+ z.coerce.number().int().min(0).max(100).optional(),
+ ),
+ // A provider answer whose arrays are all empty, for a page long enough to
+ // plausibly hold items, counts as a failure and falls back to the default
+ // model for that request.
+ EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: z
+ .stringbool()
+ .optional()
+ .default(true),
EXTRACTION_PROVIDER_TEAM_IDS: z.string().optional(),
// Let any team pick the provider per request (json format `model`); off =
// only teams in TEAM_IDS may.
@@ -852,17 +864,21 @@ const configSchema = z.object({
});
const validatedConfigSchema = configSchema.superRefine((value, context) => {
- if (
- value.EXTRACTION_PROVIDER_ROLLOUT_PERCENT +
- value.EXTRACTION_PROVIDER_SHADOW_PERCENT >
- 100
- ) {
- context.addIssue({
- code: "custom",
- path: ["EXTRACTION_PROVIDER_SHADOW_PERCENT"],
- message:
- "EXTRACTION_PROVIDER_ROLLOUT_PERCENT + EXTRACTION_PROVIDER_SHADOW_PERCENT must not exceed 100",
- });
+ for (const key of [
+ "EXTRACTION_PROVIDER_ROLLOUT_PERCENT",
+ "EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT",
+ ] as const) {
+ const rollout = value[key];
+ if (
+ rollout !== undefined &&
+ rollout + value.EXTRACTION_PROVIDER_SHADOW_PERCENT > 100
+ ) {
+ context.addIssue({
+ code: "custom",
+ path: ["EXTRACTION_PROVIDER_SHADOW_PERCENT"],
+ message: `${key} + EXTRACTION_PROVIDER_SHADOW_PERCENT must not exceed 100`,
+ });
+ }
}
for (const error of getMcpActionLogConfigErrors(value)) {
context.addIssue({
diff --git a/apps/api/src/lib/branding/font-names.ts b/apps/api/src/lib/branding/font-names.ts
index 18664fb5a..5e0925263 100644
--- a/apps/api/src/lib/branding/font-names.ts
+++ b/apps/api/src/lib/branding/font-names.ts
@@ -39,19 +39,16 @@ const SYSTEM_STACK_HEADS = new Set([
* real name further down the stack: Wix's `wfont_<site>_<hash>` and
* `wf_<hash>`, then `orig_<family>`. The alias says nothing about the font.
*/
-export const isHashedAlias = (family: string): boolean =>
+const isHashedAlias = (family: string): boolean =>
/^wfont_[0-9a-f]{6}_[0-9a-f]{32}$/i.test(family) ||
/^wf_[0-9a-f]{20,}$/i.test(family) ||
/^[a-z]{1,6}_[0-9a-f]{24,}$/i.test(family);
const unquote = (family: string) =>
- family.trim().replace(/^["']|["']$/g, "").trim();
-
-/** A stack that asks for the visitor's system font. */
-export const isSystemStack = (stack: readonly string[]): boolean => {
- const first = stack.map(unquote).find(Boolean);
- return !!first && SYSTEM_STACK_HEADS.has(first.toLowerCase());
-};
+ family
+ .trim()
+ .replace(/^["']|["']$/g, "")
+ .trim();
/**
* What a stack is set in when it names no brand face: the system font
@@ -135,7 +132,10 @@ export const titleCase = (name: string): string =>
* floor the processor's own output also meets.
*/
export function displayFamily(name: string): string {
- let out = normalizeVendorName(name).replace(/_/g, " ").replace(/\s+/g, " ").trim();
+ let out = normalizeVendorName(name)
+ .replace(/_/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
if (out && out === out.toLowerCase()) out = titleCase(out);
return knownFamily(out) ?? out;
}
diff --git a/apps/api/src/lib/extraction-provider.test.ts b/apps/api/src/lib/extraction-provider.test.ts
index 1a4dfd473..9a9371a6c 100644
--- a/apps/api/src/lib/extraction-provider.test.ts
+++ b/apps/api/src/lib/extraction-provider.test.ts
@@ -2,6 +2,7 @@ import { config } from "../config";
import {
cohortBucket,
extractionProviderCost,
+ isEmptyListAnswer,
isRequestedModelSupported,
selectExtractionRoute,
} from "./extraction-provider";
@@ -198,3 +199,80 @@ describe("extraction provider routing", () => {
);
});
});
+
+describe("ref-schema rollout", () => {
+ const provider = {
+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
+ EXTRACTION_PROVIDER_TEAM_IDS: undefined,
+ EXTRACTION_PROVIDER_SHADOW_PERCENT: 0,
+ };
+
+ it("uses REF_SCHEMA_ROLLOUT_PERCENT for ref schemas and ROLLOUT_PERCENT otherwise", () => {
+ withConfig(
+ {
+ ...provider,
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,
+ EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: 100,
+ },
+ () => {
+ expect(selectExtractionRoute({ ...base, refSchema: true })).toBe(
+ "provider",
+ );
+ expect(selectExtractionRoute({ ...base, refSchema: false })).toBe(
+ "default",
+ );
+ expect(selectExtractionRoute(base)).toBe("default");
+ },
+ );
+ });
+
+ it("falls back to ROLLOUT_PERCENT when the ref percentage is unset", () => {
+ withConfig(
+ {
+ ...provider,
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 100,
+ EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: undefined,
+ },
+ () =>
+ expect(selectExtractionRoute({ ...base, refSchema: true })).toBe(
+ "provider",
+ ),
+ );
+ });
+
+ it("keeps the shadow band directly above the ref rollout band", () => {
+ withConfig(
+ {
+ ...provider,
+ EXTRACTION_PROVIDER_ROLLOUT_PERCENT: 0,
+ EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT: 20,
+ EXTRACTION_PROVIDER_SHADOW_PERCENT: 10,
+ },
+ () => {
+ for (let i = 0; i < 300; i++) {
+ const teamId = `team-${i}`;
+ const b = cohortBucket(teamId);
+ expect(
+ selectExtractionRoute({ ...base, teamId, refSchema: true }),
+ ).toBe(b < 20 ? "provider" : b < 30 ? "shadow" : "default");
+ expect(selectExtractionRoute({ ...base, teamId })).toBe(
+ b < 10 ? "shadow" : "default",
+ );
+ }
+ },
+ );
+ });
+});
+
+describe("isEmptyListAnswer", () => {
+ it("is true only when the answer has arrays and all of them are empty", () => {
+ expect(isEmptyListAnswer({ jobs: [] })).toBe(true);
+ expect(isEmptyListAnswer({ page: { items: [], tags: [] } })).toBe(true);
+ expect(isEmptyListAnswer([])).toBe(true);
+ expect(isEmptyListAnswer({ jobs: [{ title: "Cook" }] })).toBe(false);
+ expect(isEmptyListAnswer({ jobs: [], seen: [1] })).toBe(false);
+ expect(isEmptyListAnswer({ title: "No lists here" })).toBe(false);
+ expect(isEmptyListAnswer(undefined)).toBe(false);
+ expect(isEmptyListAnswer(null)).toBe(false);
+ });
+});
diff --git a/apps/api/src/lib/extraction-provider.ts b/apps/api/src/lib/extraction-provider.ts
index 32ee13c93..971f54512 100644
--- a/apps/api/src/lib/extraction-provider.ts
+++ b/apps/api/src/lib/extraction-provider.ts
@@ -14,6 +14,12 @@ import { config } from "../config";
* A request may also name the provider model explicitly (json format `model`);
* that routes it regardless of cohort when the team is listed in TEAM_IDS or
* REQUEST_OPT_IN is on. Unknown model names are rejected at validation.
+ *
+ * Schemas with $ref/$defs can roll out on their own percentage
+ * (REF_SCHEMA_ROLLOUT_PERCENT), since the default path prices them
+ * differently. An answer that is nothing but empty arrays on a page long
+ * enough to hold items is treated as a provider failure (see
+ * isEmptyListAnswer) so the request falls back instead of returning nothing.
*/
type ExtractionRoute = "provider" | "shadow" | "default";
@@ -48,6 +54,8 @@ export function selectExtractionRoute(input: {
zeroDataRetention: boolean;
inputBytes: number;
requestedModel?: string;
+ /** The schema uses $ref/$defs (routed by REF_SCHEMA_ROLLOUT_PERCENT when set). */
+ refSchema?: boolean;
}): ExtractionRoute {
if (!isExtractionProviderConfigured()) return "default";
if (input.zeroDataRetention) return "default";
@@ -63,16 +71,41 @@ export function selectExtractionRoute(input: {
}
if (listed) return "provider";
const bucket = cohortBucket(input.teamId);
- if (bucket < config.EXTRACTION_PROVIDER_ROLLOUT_PERCENT) return "provider";
- if (
- bucket <
- config.EXTRACTION_PROVIDER_ROLLOUT_PERCENT +
- config.EXTRACTION_PROVIDER_SHADOW_PERCENT
- )
+ const rollout =
+ input.refSchema &&
+ config.EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT !== undefined
+ ? config.EXTRACTION_PROVIDER_REF_SCHEMA_ROLLOUT_PERCENT
+ : config.EXTRACTION_PROVIDER_ROLLOUT_PERCENT;
+ if (bucket < rollout) return "provider";
+ if (bucket < rollout + config.EXTRACTION_PROVIDER_SHADOW_PERCENT)
return "shadow";
return "default";
}
+/** Pages shorter than this can legitimately have no list items. */
+export const EMPTY_LIST_FALLBACK_MIN_CHARS = 2000;
+
+function arraysIn(value: unknown, out: unknown[][] = []): unknown[][] {
+ if (Array.isArray(value)) {
+ out.push(value);
+ for (const v of value) arraysIn(v, out);
+ } else if (value && typeof value === "object") {
+ for (const v of Object.values(value as Record<string, unknown>))
+ arraysIn(v, out);
+ }
+ return out;
+}
+
+/**
+ * True when the answer contains arrays and every one of them is empty — the
+ * shape a model produces when it gives up on a listing page. Only meaningful
+ * with a markdown input of at least EMPTY_LIST_FALLBACK_MIN_CHARS.
+ */
+export function isEmptyListAnswer(answer: unknown): boolean {
+ const arrays = arraysIn(answer);
+ return arrays.length > 0 && arrays.every(a => a.length === 0);
+}
+
/** Per-million-token prices for the provider model (amortized infrastructure cost, configured). */
export function extractionProviderCost(
model: string,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
index 25950002b..4d02ab07a 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.test.ts
@@ -19,6 +19,7 @@ import { performSummary } from "./llmExtract";
import { performCleanContent } from "./llmExtract";
import { encoding_for_model } from "@dqbd/tiktoken";
import type { Mock } from "vitest";
+import { config } from "../../../config";
vi.mock("@dqbd/tiktoken", () => ({
encoding_for_model: vi.fn(),
@@ -482,3 +483,127 @@ describe("performLLMExtract empty-result handling", () => {
);
});
});
+
+describe("performLLMExtract extraction provider fallback", () => {
+ const listSchema = {
+ type: "object",
+ properties: {
+ jobs: {
+ type: "array",
+ items: { type: "object", properties: { title: { type: "string" } } },
+ },
+ },
+ };
+ const longPage = "Open roles: Line cook. Pizza cook. Apply now. ".repeat(60); // > 2,000 chars
+ const saved: Partial<typeof config> = {};
+
+ function withProvider(overrides: Partial<typeof config> = {}) {
+ const values: Partial<typeof config> = {
+ EXTRACTION_PROVIDER_BASE_URL: "http://provider.local/v1",
+ EXTRACTION_PROVIDER_MODEL: "extraction-v1",
+ EXTRACTION_PROVIDER_TEAM_IDS: "test-team",
+ EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: true,
+ ...overrides,
+ };
+ for (const k of Object.keys(values) as (keyof typeof config)[]) {
+ (saved as any)[k] = config[k];
+ (config as any)[k] = values[k];
+ }
+ }
+
+ function meta() {
+ const m = makeJsonMeta();
+ m.options.formats[0].schema = listSchema as any;
+ (m.logger as any).warn = vi.fn(); // the fallback path logs a warning
+ return m;
+ }
+
+ function modelIds() {
+ return (extractData as Mock).mock.calls.map(
+ c => c[0].extractOptions.model.modelId,
+ );
+ }
+
+ beforeEach(() => {
+ (extractData as Mock).mockReset();
+ });
+
+ afterEach(() => {
+ Object.assign(config, saved);
+ for (const k of Object.keys(saved)) delete (saved as any)[k];
+ });
+
+ it("falls back to the default model when the provider answers with only empty lists on a long page", async () => {
+ withProvider();
+ (extractData as Mock)
+ .mockResolvedValueOnce({
+ extractedDataArray: [{ jobs: [] }],
+ warning: undefined,
+ costLimitExceededTokenUsage: null,
+ })
+ .mockResolvedValueOnce({
+ extractedDataArray: [{ jobs: [{ title: "Line cook" }] }],
+ warning: undefined,
+ costLimitExceededTokenUsage: null,
+ });
+
+ const result = await performLLMExtract(meta(), {
+ markdown: longPage,
+ metadata: {},
+ } as any);
+
+ expect(modelIds()).toEqual(["extraction-v1", "gpt-4o-mini"]);
+ expect(result.json).toEqual({ jobs: [{ title: "Line cook" }] });
+ });
+
+ it("keeps the provider's empty lists on a short page", async () => {
+ withProvider();
+ (extractData as Mock).mockResolvedValueOnce({
+ extractedDataArray: [{ jobs: [] }],
+ warning: undefined,
+ costLimitExceededTokenUsage: null,
+ });
+
+ const result = await performLLMExtract(meta(), {
+ markdown: "No openings right now.",
+ metadata: {},
+ } as any);
+
+ expect(modelIds()).toEqual(["extraction-v1"]);
+ expect(result.json).toEqual({ jobs: [] });
+ });
+
+ it("keeps the provider's empty lists when the fallback is switched off", async () => {
+ withProvider({ EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS: false });
+ (extractData as Mock).mockResolvedValueOnce({
+ extractedDataArray: [{ jobs: [] }],
+ warning: undefined,
+ costLimitExceededTokenUsage: null,
+ });
+
+ const result = await performLLMExtract(meta(), {
+ markdown: longPage,
+ metadata: {},
+ } as any);
+
+ expect(modelIds()).toEqual(["extraction-v1"]);
+ expect(result.json).toEqual({ jobs: [] });
+ });
+
+ it("does not fall back when the provider found items", async () => {
+ withProvider();
+ (extractData as Mock).mockResolvedValueOnce({
+ extractedDataArray: [{ jobs: [{ title: "Pizza cook" }] }],
+ warning: undefined,
+ costLimitExceededTokenUsage: null,
+ });
+
+ const result = await performLLMExtract(meta(), {
+ markdown: longPage,
+ metadata: {},
+ } as any);
+
+ expect(modelIds()).toEqual(["extraction-v1"]);
+ expect(result.json).toEqual({ jobs: [{ title: "Pizza cook" }] });
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index b253c3b63..8fe6491fa 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -33,7 +33,9 @@ import { isZeroDataRetentionActive } from "../../../lib/otel-tracer";
import { isAgentExtractModelValid } from "../../../controllers/v1/types";
import { hasFormatOfType } from "../../../lib/format-utils";
import {
+ EMPTY_LIST_FALLBACK_MIN_CHARS,
extractionProviderCost,
+ isEmptyListAnswer,
selectExtractionRoute,
} from "../../../lib/extraction-provider";
import { config } from "../../../config";
@@ -1105,6 +1107,7 @@ export async function performLLMExtract(
isZeroDataRetentionActive(),
inputBytes: Buffer.byteLength(document.markdown ?? "", "utf8"),
requestedModel: jsonFormat.model,
+ refSchema: modelSelection.reason === "recursive_schema_detected",
});
if (jsonFormat.model !== undefined && route !== "provider") {
document.warning =
@@ -1143,6 +1146,14 @@ export async function performLLMExtract(
attempt = await extractWith(providerOptions());
if (attempt.warning?.includes(PROVIDER_FAILURE)) {
failure = attempt.warning;
+ } else if (
+ config.EXTRACTION_PROVIDER_FALLBACK_ON_EMPTY_LISTS &&
+ (document.markdown?.length ?? 0) >= EMPTY_LIST_FALLBACK_MIN_CHARS &&
+ isEmptyListAnswer(
+ attempt.extractedDataArray[attempt.extractedDataArray.length - 1],
+ )
+ ) {
+ failure = "provider returned only empty lists";
}
} catch (error) {
failure = (error as Error).message;