Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(json): stop JSON extraction failing on schema wrapping, oversized pages and unsupported schemas (#4807)

Gergő Móricz2c88cbb041cd
diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.ts
index 0e98bf04d..89dedbb10 100644
--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts
@@ -985,7 +985,7 @@ describe("V1 Types Validation", () => {
schema: {
type: "object",
properties: {
- changes: { type: "array" },
+ changes: { type: "array", items: { type: "string" } },
},
},
modes: ["json"],
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index dbd201e44..b787a2c89 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -1685,6 +1685,34 @@ describe("Scrape tests", () => {
);
});
+ it.concurrent(
+ "rejects JSON schemas that structured outputs cannot express",
+ async () => {
+ const raw = await scrapeRaw(
+ {
+ url: base,
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: { events: { type: "array" } },
+ },
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe(
+ 'Invalid JSON schema at "properties.events": arrays must define "items".',
+ );
+ },
+ scrapeTimeout,
+ );
+
describeIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
"JSON format",
() => {
@@ -1736,6 +1764,54 @@ describe("Scrape tests", () => {
},
scrapeTimeout,
);
+
+ it.concurrent(
+ "works with a bare map of property names to schemas",
+ async () => {
+ const response = await scrape(
+ {
+ url: base,
+ formats: [
+ {
+ type: "json",
+ schema: {
+ company_name: { type: "string" },
+ is_open_source: { type: "boolean" },
+ },
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.warning ?? "").not.toContain(
+ "JSON extraction failed",
+ );
+ expect(typeof response.json.company_name).toBe("string");
+ expect(typeof response.json.is_open_source).toBe("boolean");
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "works without a schema or prompt",
+ async () => {
+ const response = await scrape(
+ {
+ url: base,
+ formats: [{ type: "json" }],
+ },
+ identity,
+ );
+
+ expect(response.warning ?? "").not.toContain(
+ "JSON extraction failed",
+ );
+ expect(response.json).not.toBeNull();
+ expect(typeof response.json).toBe("object");
+ },
+ scrapeTimeout,
+ );
},
);
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index 36dff1e21..48bc79bfe 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -149,6 +149,72 @@ describe("V2 Types Validation", () => {
expect(result.timeout).toBe(60000); // Should be transformed from 30000
});
+ it("should reject JSON schemas structured outputs cannot express", () => {
+ const input: ScrapeRequestInput = {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: { events: { type: "array" } },
+ },
+ },
+ ],
+ };
+
+ const result = scrapeRequestSchema.safeParse(input);
+ expect(result.success).toBe(false);
+ expect(result.error?.issues.map(issue => issue.message)).toContain(
+ 'Invalid JSON schema at "properties.events": arrays must define "items".',
+ );
+ });
+
+ it("should accept JSON schemas with capitalized types and oneOf", () => {
+ const input: ScrapeRequestInput = {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: {
+ name: { type: "String" },
+ slot: { oneOf: [{ type: "string" }, { type: "number" }] },
+ },
+ },
+ },
+ ],
+ };
+
+ expect(() => scrapeRequestSchema.parse(input)).not.toThrow();
+ });
+
+ it("should reject unsupported changeTracking schemas", () => {
+ const input: ScrapeRequestInput = {
+ url: "https://example.com",
+ formats: [
+ "markdown",
+ {
+ type: "changeTracking",
+ modes: ["json"],
+ schema: {
+ type: "object",
+ properties: {
+ price: { type: "object", properties: {}, allOf: [] },
+ },
+ },
+ },
+ ],
+ };
+
+ const result = scrapeRequestSchema.safeParse(input);
+ expect(result.success).toBe(false);
+ expect(result.error?.issues.map(issue => issue.message)).toContain(
+ 'Invalid JSON schema at "properties.price": "allOf" is not supported for JSON extraction.',
+ );
+ });
+
it("should accept query format without markdown", () => {
const input: ScrapeRequestInput = {
url: "https://example.com",
@@ -245,7 +311,7 @@ describe("V2 Types Validation", () => {
schema: {
type: "object",
properties: {
- changes: { type: "array" },
+ changes: { type: "array", items: { type: "string" } },
},
},
modes: ["json"],
@@ -805,6 +871,21 @@ describe("V2 Types Validation", () => {
expect(result.urls).toHaveLength(10);
});
+ it("should reject JSON schemas structured outputs cannot express", () => {
+ const result = extractRequestSchema.safeParse({
+ urls: ["https://example.com"],
+ schema: {
+ type: "object",
+ properties: { events: { type: "array" } },
+ },
+ });
+
+ expect(result.success).toBe(false);
+ expect(result.error?.issues.map(issue => issue.message)).toContain(
+ 'Invalid JSON schema at "properties.events": arrays must define "items".',
+ );
+ });
+
it("should reject invalid JSON schema", () => {
const input: ExtractRequestInput = {
urls: ["https://example.com"],
@@ -1984,7 +2065,7 @@ describe("V2 Types Validation", () => {
schema: {
type: "object",
properties: {
- changes: { type: "array" },
+ changes: { type: "array", items: { type: "string" } },
},
},
},
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index ec34c63eb..9d6386358 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -5,6 +5,7 @@ import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
import { hasReachableHost } from "../../lib/url-utils";
import { countries } from "../../lib/validate-country";
import { addPathRegexIssues, pathPatternsSchema } from "../../lib/crawl-regex";
+import { addStrictSchemaIssue } from "../../lib/openai-strict-schema";
import type { PdfPageBlocks } from "../../scraper/scrapeURL/engines/pdf/types";
import {
ExtractorOptions,
@@ -219,7 +220,8 @@ export const extractOptions = z
.transform(val => normalizeSchemaForOpenAI(val))
.refine(val => validateSchemaForOpenAI(val), {
message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ })
+ .superRefine(addStrictSchemaIssue),
systemPrompt: z.string().max(10000).prefault(""),
prompt: z.string().max(10000).optional(),
temperature: z.number().optional(),
@@ -254,7 +256,8 @@ const extractOptionsWithAgent = z
.transform(val => normalizeSchemaForOpenAI(val))
.refine(val => validateSchemaForOpenAI(val), {
message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ })
+ .superRefine(addStrictSchemaIssue),
systemPrompt: z.string().max(10000).prefault(""),
prompt: z.string().max(10000).optional(),
temperature: z.number().optional(),
@@ -509,7 +512,8 @@ const baseScrapeOptions = z.strictObject({
.transform(val => normalizeSchemaForOpenAI(val))
.refine(val => validateSchemaForOpenAI(val), {
message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ })
+ .superRefine(addStrictSchemaIssue),
modes: z.enum(["json", "git-diff"]).array().optional().prefault([]),
tag: z.string().or(z.null()).prefault(null),
})
@@ -765,7 +769,8 @@ const extractV1Options = z
.transform(val => normalizeSchemaForOpenAI(val))
.refine(val => validateSchemaForOpenAI(val), {
message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ })
+ .superRefine(addStrictSchemaIssue),
limit: z.int().positive().finite().optional(),
ignoreSitemap: z.boolean().prefault(false),
includeSubdomains: z.boolean().prefault(true),
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 515a5ed7a..18bf6ea39 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -8,6 +8,7 @@ import { hasReachableHost } from "../../lib/url-utils";
import { countries } from "../../lib/validate-country";
import { includesFormat } from "../../lib/format-utils";
import { addPathRegexIssues, pathPatternsSchema } from "../../lib/crawl-regex";
+import { addStrictSchemaIssue } from "../../lib/openai-strict-schema";
import {
ExtractorOptions,
PageOptions,
@@ -359,7 +360,8 @@ const jsonFormatWithOptions = z.strictObject({
.transform(val => normalizeSchemaForOpenAI(val))
.refine(val => validateSchemaForOpenAI(val), {
message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ })
+ .superRefine(addStrictSchemaIssue),
prompt: z.string().max(10000).optional(),
checkPromptInjection: z.boolean().optional(),
});
@@ -394,7 +396,8 @@ const changeTrackingFormatWithOptions = z.strictObject({
.transform(val => normalizeSchemaForOpenAI(val))
.refine(val => validateSchemaForOpenAI(val), {
message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ })
+ .superRefine(addStrictSchemaIssue),
modes: z.enum(["json", "git-diff"]).array().optional().prefault([]),
tag: z.string().or(z.null()).prefault(null),
});
@@ -1004,7 +1007,8 @@ const extractOptions = z
.transform(val => normalizeSchemaForOpenAI(val))
.refine(val => validateSchemaForOpenAI(val), {
message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ })
+ .superRefine(addStrictSchemaIssue),
limit: z.int().positive().finite().optional(),
ignoreSitemap: z.boolean().prefault(false),
includeSubdomains: z.boolean().prefault(true),
diff --git a/apps/api/src/lib/openai-strict-schema.test.ts b/apps/api/src/lib/openai-strict-schema.test.ts
new file mode 100644
index 000000000..86ce879b2
--- /dev/null
+++ b/apps/api/src/lib/openai-strict-schema.test.ts
@@ -0,0 +1,450 @@
+import { z } from "zod";
+import {
+ addStrictSchemaIssue,
+ findStrictSchemaViolation,
+ normalizeSchemaKeywords,
+ toRootSchema,
+} from "./openai-strict-schema";
+
+describe("normalizeSchemaKeywords", () => {
+ it("lowercases type names, including nested and nullable ones", () => {
+ const result = normalizeSchemaKeywords({
+ type: "Object",
+ properties: {
+ name: { type: "String" },
+ count: { type: ["Integer", "null"] },
+ tags: { type: "Array", items: { type: "String" } },
+ },
+ });
+
+ expect(result).toEqual({
+ type: "object",
+ properties: {
+ name: { type: "string" },
+ count: { type: ["integer", "null"] },
+ tags: { type: "array", items: { type: "string" } },
+ },
+ });
+ });
+
+ it("turns oneOf into anyOf", () => {
+ const result = normalizeSchemaKeywords({
+ type: "object",
+ properties: {
+ slot: { oneOf: [{ type: "string" }, { type: "Number" }] },
+ },
+ });
+
+ expect(result.properties.slot).toEqual({
+ anyOf: [{ type: "string" }, { type: "number" }],
+ });
+ });
+
+ it("infers a missing type from properties or items below the root", () => {
+ const result = normalizeSchemaKeywords({
+ type: "object",
+ properties: {
+ identity: { properties: { name: { type: "string" } } },
+ list: { items: { type: "string" } },
+ },
+ });
+
+ expect(result.properties.identity.type).toBe("object");
+ expect(result.properties.list.type).toBe("array");
+ });
+
+ it("leaves a root map of property names alone, even with an 'items' field", () => {
+ const schema = {
+ items: { type: "array", items: { type: "string" } },
+ total: { type: "number" },
+ };
+
+ expect(normalizeSchemaKeywords(schema)).toEqual(schema);
+ });
+
+ it("infers object for a typeless root that only uses schema keywords", () => {
+ const result = normalizeSchemaKeywords({
+ $schema: "http://json-schema.org/draft-07/schema#",
+ properties: { name: { type: "string" } },
+ required: ["name"],
+ });
+
+ expect(result.type).toBe("object");
+ });
+
+ it("drops additionalProperties schemas next to properties", () => {
+ const result = normalizeSchemaKeywords({
+ type: "object",
+ properties: { name: { type: "string" } },
+ additionalProperties: { type: "array" },
+ });
+
+ expect(result.additionalProperties).toBe(false);
+ });
+
+ it("does not modify its input", () => {
+ const schema = { type: "String" };
+ normalizeSchemaKeywords(schema);
+ expect(schema.type).toBe("String");
+ });
+});
+
+describe("toRootSchema", () => {
+ it("turns a bare map into an object schema, skipping $schema", () => {
+ expect(
+ toRootSchema({
+ $schema: "http://json-schema.org/draft-07/schema#",
+ name: { type: "String" },
+ }),
+ ).toEqual({
+ type: "object",
+ properties: { name: { type: "string" } },
+ required: ["name"],
+ additionalProperties: false,
+ });
+ });
+
+ it("keeps $-named fields other than annotations, and moves $defs to the root", () => {
+ expect(
+ toRootSchema({
+ $schema: "http://json-schema.org/draft-07/schema#",
+ $metadata: { type: "string" },
+ item: { $ref: "#/$defs/item" },
+ $defs: { item: { type: "object", properties: {} } },
+ }),
+ ).toEqual({
+ type: "object",
+ properties: {
+ $metadata: { type: "string" },
+ item: { $ref: "#/$defs/item" },
+ },
+ required: ["$metadata", "item"],
+ additionalProperties: false,
+ $defs: { item: { type: "object", properties: {} } },
+ });
+ });
+
+ it("treats a field named 'properties' as a field, not as a schema keyword", () => {
+ const result = toRootSchema({
+ title: { type: "string" },
+ properties: {
+ type: "array",
+ items: { type: "object", properties: { price: { type: "number" } } },
+ },
+ });
+
+ expect(result.type).toBe("object");
+ expect(Object.keys(result.properties)).toEqual(["title", "properties"]);
+ expect(result.properties.properties.type).toBe("array");
+ });
+
+ it("keeps a typeless root schema a schema", () => {
+ const result = toRootSchema({
+ title: "Listing",
+ properties: { price: { type: "number" } },
+ required: ["price"],
+ });
+
+ expect(result).toEqual({
+ title: "Listing",
+ type: "object",
+ properties: { price: { type: "number" } },
+ required: ["price"],
+ });
+ });
+});
+
+describe("findStrictSchemaViolation", () => {
+ it("accepts a plain object schema", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ tags: { type: "array", items: { type: "string" } },
+ status: { type: "string", enum: ["open", "closed"] },
+ price: { anyOf: [{ type: "number" }, { type: "null" }] },
+ },
+ required: ["title"],
+ }),
+ ).toBeNull();
+ });
+
+ it("accepts undefined (no schema)", () => {
+ expect(findStrictSchemaViolation(undefined)).toBeNull();
+ });
+
+ it("accepts what normalizeSchemaKeywords fixes", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: {
+ name: { type: "String" },
+ slot: { oneOf: [{ type: "string" }, { type: "integer" }] },
+ identity: { properties: { value: { type: "string" } } },
+ },
+ }),
+ ).toBeNull();
+ });
+
+ it("accepts a root map of property names to schemas", () => {
+ expect(
+ findStrictSchemaViolation({
+ $schema: "http://json-schema.org/draft-07/schema#",
+ judgments: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: { citation: { type: "string" } },
+ },
+ },
+ }),
+ ).toBeNull();
+ });
+
+ it("accepts a bare map with a field named 'properties'", () => {
+ expect(
+ findStrictSchemaViolation({
+ title: { type: "string" },
+ properties: { type: "array", items: { type: "string" } },
+ }),
+ ).toBeNull();
+ });
+
+ it("accepts nullable root arrays", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: ["array", "null"],
+ items: { type: "string" },
+ }),
+ ).toBeNull();
+ });
+
+ it("rejects roots that are neither objects nor arrays", () => {
+ expect(findStrictSchemaViolation({ type: "string" })).toBe(
+ 'Invalid JSON schema: the root must be an object or array schema, got type "string".',
+ );
+ expect(
+ findStrictSchemaViolation({
+ anyOf: [{ type: "object", properties: {} }, { type: "null" }],
+ }),
+ ).toBe(
+ 'Invalid JSON schema: the root must be an object or array schema, not "anyOf".',
+ );
+ });
+
+ it("checks definitions behind a root $ref", () => {
+ expect(
+ findStrictSchemaViolation({
+ $ref: "#/$defs/listing",
+ $defs: {
+ listing: {
+ type: "object",
+ properties: { photos: { type: "array" } },
+ },
+ },
+ }),
+ ).toBe(
+ 'Invalid JSON schema at "$defs.listing.properties.photos": arrays must define "items".',
+ );
+ });
+
+ it("checks keywords next to a $ref", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: { node: { $ref: "#/$defs/node", allOf: [] } },
+ $defs: { node: { type: "object", properties: {} } },
+ }),
+ ).toBe(
+ 'Invalid JSON schema at "properties.node": "allOf" is not supported for JSON extraction.',
+ );
+ });
+
+ it("accepts recursive schemas using $ref", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: { root: { $ref: "#/$defs/node" } },
+ $defs: {
+ node: {
+ type: "object",
+ properties: {
+ children: { type: "array", items: { $ref: "#/$defs/node" } },
+ },
+ },
+ },
+ }),
+ ).toBeNull();
+ });
+
+ it("rejects arrays without items, naming the path", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: {
+ centers: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: { professionals: { type: "array" } },
+ },
+ },
+ },
+ }),
+ ).toBe(
+ 'Invalid JSON schema at "properties.centers.items.properties.professionals": arrays must define "items".',
+ );
+ });
+
+ it("rejects tuple-style items", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: {
+ row: {
+ type: "array",
+ items: [{ type: "string" }, { type: "number" }],
+ },
+ },
+ }),
+ ).toContain('"items" must be a single schema object');
+ });
+
+ it.each([
+ "allOf",
+ "not",
+ "if",
+ "then",
+ "else",
+ "dependentRequired",
+ "dependentSchemas",
+ ])("rejects %s", keyword => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: {
+ data: { type: "object", properties: {}, [keyword]: [] },
+ },
+ }),
+ ).toBe(
+ `Invalid JSON schema at "properties.data": "${keyword}" is not supported for JSON extraction.`,
+ );
+ });
+
+ it("rejects oneOf when anyOf is also present", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: {
+ slot: {
+ anyOf: [{ type: "string" }],
+ oneOf: [{ type: "number" }],
+ },
+ },
+ }),
+ ).toBe(
+ 'Invalid JSON schema at "properties.slot": "oneOf" is not supported for JSON extraction.',
+ );
+ });
+
+ it("rejects non-object properties and non-array anyOf", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: { data: { type: "object", properties: ["a", "b"] } },
+ }),
+ ).toBe(
+ 'Invalid JSON schema at "properties.data": "properties" must be an object, got ["a","b"].',
+ );
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: { slot: { anyOf: { type: "string" } } },
+ }),
+ ).toBe(
+ 'Invalid JSON schema at "properties.slot": "anyOf" must be an array, got {"type":"string"}.',
+ );
+ });
+
+ it("ignores additionalProperties schemas next to properties", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: { name: { type: "string" } },
+ additionalProperties: { type: "array" },
+ }),
+ ).toBeNull();
+ });
+
+ it("rejects typed dictionaries", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: {
+ prices: { type: "object", additionalProperties: { type: "number" } },
+ },
+ }),
+ ).toBe(
+ 'Invalid JSON schema at "properties.prices": objects must list their "properties"; a dictionary described only by "additionalProperties" is not supported for JSON extraction.',
+ );
+ });
+
+ it("rejects type names that are not JSON Schema types", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: { level: { type: "enum", enum: ["a", "b"] } },
+ }),
+ ).toContain('"type" must be one of');
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: { title: { type: { type: "string" } } },
+ }),
+ ).toContain('"type" must be one of');
+ });
+
+ it("rejects shorthand property values", () => {
+ expect(
+ findStrictSchemaViolation({ status: "string", price: "string" }),
+ ).toBe(
+ 'Invalid JSON schema at "properties.status": expected a schema object, got "string".',
+ );
+ });
+
+ it("rejects nested schemas without a type", () => {
+ expect(
+ findStrictSchemaViolation({
+ type: "object",
+ properties: { metadata: {} },
+ }),
+ ).toBe('Invalid JSON schema at "properties.metadata": missing "type".');
+ });
+});
+
+describe("addStrictSchemaIssue", () => {
+ const schemaField = z.any().optional().superRefine(addStrictSchemaIssue);
+
+ it("adds a zod issue for an unsupported schema", () => {
+ const result = schemaField.safeParse({
+ type: "object",
+ properties: { events: { type: "array" } },
+ });
+
+ expect(result.success).toBe(false);
+ expect(result.error?.issues[0].message).toBe(
+ 'Invalid JSON schema at "properties.events": arrays must define "items".',
+ );
+ });
+
+ it("passes supported and missing schemas", () => {
+ expect(schemaField.safeParse(undefined).success).toBe(true);
+ expect(
+ schemaField.safeParse({
+ type: "object",
+ properties: { title: { type: "string" } },
+ }).success,
+ ).toBe(true);
+ });
+});
diff --git a/apps/api/src/lib/openai-strict-schema.ts b/apps/api/src/lib/openai-strict-schema.ts
new file mode 100644
index 000000000..cfebfa714
--- /dev/null
+++ b/apps/api/src/lib/openai-strict-schema.ts
@@ -0,0 +1,315 @@
+import { z } from "zod";
+
+// OpenAI structured outputs (strict mode) accept only a subset of JSON Schema.
+// normalizeSchemaKeywords fixes the mechanical mistakes we see in user-written
+// and LLM-generated schemas; findStrictSchemaViolation names the constructs
+// strict mode rejects outright, so they can be refused before any scrape runs.
+
+const JSON_SCHEMA_TYPES = new Set([
+ "string",
+ "number",
+ "integer",
+ "boolean",
+ "object",
+ "array",
+ "null",
+]);
+
+// Rejected by strict mode wherever they appear.
+const UNSUPPORTED_KEYWORDS = [
+ "allOf",
+ "oneOf",
+ "not",
+ "if",
+ "then",
+ "else",
+ "dependentRequired",
+ "dependentSchemas",
+];
+
+type Schema = Record<string, any>;
+
+function isPlainObject(x: unknown): x is Schema {
+ return typeof x === "object" && x !== null && !Array.isArray(x);
+}
+
+function mapValues(obj: Schema, fn: (v: any) => any): Schema {
+ return Object.fromEntries(Object.entries(obj).map(([k, v]) => [k, fn(v)]));
+}
+
+function lowercaseType(type: unknown): unknown {
+ if (typeof type === "string") return type.toLowerCase();
+ if (Array.isArray(type)) {
+ return type.map(t => (typeof t === "string" ? t.toLowerCase() : t));
+ }
+ return type;
+}
+
+export function typeIncludes(type: unknown, name: string): boolean {
+ return type === name || (Array.isArray(type) && type.includes(name));
+}
+
+// Whether a typeless root is a schema rather than a bare map of property names
+// to schemas (which JSON extraction also accepts at the root): every key is a
+// schema keyword holding the kind of value that keyword takes, and
+// "properties" maps names to schemas instead of being a schema itself (a
+// field that happens to be called "properties").
+function isTypelessRootSchema(node: Schema): boolean {
+ if (!isPlainObject(node.properties)) return false;
+ if (typeof node.properties.type === "string") return false;
+ return Object.entries(node).every(([key, value]) => {
+ switch (key) {
+ case "$schema":
+ case "$id":
+ case "$comment":
+ case "title":
+ case "description":
+ return typeof value === "string";
+ case "required":
+ return Array.isArray(value);
+ case "additionalProperties":
+ return typeof value === "boolean";
+ case "properties":
+ case "$defs":
+ case "definitions":
+ return isPlainObject(value);
+ default:
+ return false;
+ }
+ });
+}
+
+function isBareRootMap(node: unknown): node is Schema {
+ return (
+ isPlainObject(node) &&
+ node.type === undefined &&
+ node.anyOf === undefined &&
+ node.oneOf === undefined &&
+ node.$ref === undefined &&
+ !isTypelessRootSchema(node)
+ );
+}
+
+/**
+ * Lowercases type names ("String" -> "string"), turns oneOf into anyOf, and
+ * fills in a missing type when properties/items make it unambiguous. Returns
+ * a new schema; the input is not modified.
+ */
+export function normalizeSchemaKeywords(schema: any): any {
+ return normalizeNode(schema, true);
+}
+
+function normalizeNode(node: any, isRoot: boolean): any {
+ if (!isPlainObject(node)) return node;
+ const out: Schema = { ...node };
+
+ if (out.type !== undefined) {
+ out.type = lowercaseType(out.type);
+ }
+
+ if (Array.isArray(out.oneOf) && out.anyOf === undefined) {
+ out.anyOf = out.oneOf;
+ delete out.oneOf;
+ }
+
+ if (out.type === undefined) {
+ if (isRoot ? isTypelessRootSchema(out) : isPlainObject(out.properties)) {
+ out.type = "object";
+ } else if (!isRoot && isPlainObject(out.items)) {
+ out.type = "array";
+ }
+ }
+
+ if (isPlainObject(out.properties)) {
+ out.properties = mapValues(out.properties, v => normalizeNode(v, false));
+ }
+ if (isPlainObject(out.items)) {
+ out.items = normalizeNode(out.items, false);
+ }
+ if (Array.isArray(out.anyOf)) {
+ out.anyOf = out.anyOf.map(v => normalizeNode(v, false));
+ }
+ // Strict mode closes every object (additionalProperties: false), so keys
+ // beyond "properties" never come back; a schema for them is dropped here
+ // rather than validated. Without "properties", it's a typed dictionary,
+ // which findStrictSchemaViolation reports.
+ if (isPlainObject(out.additionalProperties)) {
+ out.additionalProperties = isPlainObject(out.properties)
+ ? false
+ : normalizeNode(out.additionalProperties, false);
+ }
+ for (const defsKey of ["$defs", "definitions"]) {
+ if (isPlainObject(out[defsKey])) {
+ out[defsKey] = mapValues(out[defsKey], v => normalizeNode(v, false));
+ }
+ }
+
+ return out;
+}
+
+// Annotations that can sit next to the fields of a bare map. Any other key,
+// "$"-prefixed or not, is a property name.
+const ROOT_ANNOTATIONS = new Set(["$schema", "$id", "$comment"]);
+
+/**
+ * normalizeSchemaKeywords, plus turning a bare map of property names to
+ * schemas into an object schema. Its "$defs" (a map of definitions, not a
+ * schema) moves to the root of that object schema.
+ */
+export function toRootSchema(schema: any): any {
+ if (!isBareRootMap(schema)) return normalizeSchemaKeywords(schema);
+ const { $defs, ...rest } = schema;
+ const hasDefs = isPlainObject($defs) && typeof $defs.type !== "string";
+ const fields = hasDefs ? rest : schema;
+ const entries = Object.entries(fields).filter(
+ ([key]) => !ROOT_ANNOTATIONS.has(key),
+ );
+ return normalizeSchemaKeywords({
+ type: "object",
+ properties: Object.fromEntries(entries),
+ required: entries.map(([key]) => key),
+ additionalProperties: false,
+ ...(hasDefs ? { $defs } : {}),
+ });
+}
+
+function describe(value: unknown): string {
+ const json = JSON.stringify(value) ?? String(value);
+ return json.length > 60 ? json.slice(0, 57) + "..." : json;
+}
+
+/**
+ * Returns a human-readable description of the first construct in the schema
+ * that OpenAI strict structured outputs rejects, or null if none was found.
+ * The schema is normalized with normalizeSchemaKeywords first, so anything
+ * that function fixes is not reported.
+ */
+export function findStrictSchemaViolation(schema: any): string | null {
+ if (schema === undefined || schema === null) return null;
+ const root = toRootSchema(schema);
+ if (!isPlainObject(root)) {
+ return `Invalid JSON schema: expected an object, got ${describe(root)}.`;
+ }
+
+ // Structured outputs need an object at the root; JSON extraction wraps a
+ // root array in one, but nothing else.
+ if (root.$ref === undefined) {
+ if (root.anyOf !== undefined) {
+ return 'Invalid JSON schema: the root must be an object or array schema, not "anyOf".';
+ }
+ if (
+ !typeIncludes(root.type, "object") &&
+ !typeIncludes(root.type, "array")
+ ) {
+ return `Invalid JSON schema: the root must be an object or array schema, got type ${describe(root.type)}.`;
+ }
+ }
+
+ return checkNode(root, "");
+}
+
+function checkNode(node: any, path: string): string | null {
+ const at = path ? ` at "${path}"` : "";
+ const child = (segment: string) => (path ? `${path}.${segment}` : segment);
+
+ if (!isPlainObject(node)) {
+ return `Invalid JSON schema${at}: expected a schema object, got ${describe(node)}.`;
+ }
+
+ for (const keyword of UNSUPPORTED_KEYWORDS) {
+ if (node[keyword] !== undefined) {
+ return `Invalid JSON schema${at}: "${keyword}" is not supported for JSON extraction.`;
+ }
+ }
+
+ // A reference takes its type from the definition it points to, which is
+ // checked where it is declared.
+ const isReference = node.$ref !== undefined;
+
+ if (
+ !isReference &&
+ node.type === undefined &&
+ node.anyOf === undefined &&
+ node.enum === undefined &&
+ node.const === undefined
+ ) {
+ return `Invalid JSON schema${at}: missing "type".`;
+ }
+
+ if (node.type !== undefined) {
+ const types = Array.isArray(node.type) ? node.type : [node.type];
+ const invalid = types.find(
+ t => typeof t !== "string" || !JSON_SCHEMA_TYPES.has(t),
+ );
+ if (types.length === 0 || invalid !== undefined) {
+ return `Invalid JSON schema${at}: "type" must be one of ${[...JSON_SCHEMA_TYPES].join(", ")}, got ${describe(node.type)}.`;
+ }
+ }
+
+ if (
+ !isReference &&
+ typeIncludes(node.type, "array") &&
+ !isPlainObject(node.items)
+ ) {
+ return node.items === undefined
+ ? `Invalid JSON schema${at}: arrays must define "items".`
+ : `Invalid JSON schema${at}: "items" must be a single schema object, got ${describe(node.items)}.`;
+ }
+
+ if (
+ typeIncludes(node.type, "object") &&
+ node.properties === undefined &&
+ isPlainObject(node.additionalProperties)
+ ) {
+ return `Invalid JSON schema${at}: objects must list their "properties"; a dictionary described only by "additionalProperties" is not supported for JSON extraction.`;
+ }
+
+ if (node.properties !== undefined) {
+ if (!isPlainObject(node.properties)) {
+ return `Invalid JSON schema${at}: "properties" must be an object, got ${describe(node.properties)}.`;
+ }
+ for (const [key, value] of Object.entries(node.properties)) {
+ const violation = checkNode(value, child(`properties.${key}`));
+ if (violation) return violation;
+ }
+ }
+
+ if (isPlainObject(node.items)) {
+ const violation = checkNode(node.items, child("items"));
+ if (violation) return violation;
+ }
+
+ if (node.anyOf !== undefined) {
+ if (!Array.isArray(node.anyOf)) {
+ return `Invalid JSON schema${at}: "anyOf" must be an array, got ${describe(node.anyOf)}.`;
+ }
+ for (let i = 0; i < node.anyOf.length; i++) {
+ const violation = checkNode(node.anyOf[i], child(`anyOf.${i}`));
+ if (violation) return violation;
+ }
+ }
+
+ for (const defsKey of ["$defs", "definitions"]) {
+ if (isPlainObject(node[defsKey])) {
+ for (const [key, value] of Object.entries(node[defsKey])) {
+ const violation = checkNode(value, child(`${defsKey}.${key}`));
+ if (violation) return violation;
+ }
+ }
+ }
+
+ return null;
+}
+
+// Zod refinement for request schemas that JSON extraction sends to OpenAI in
+// strict mode: an unsupported schema would otherwise fail mid-scrape and come
+// back as a warning on an otherwise successful response.
+export function addStrictSchemaIssue(
+ schema: unknown,
+ ctx: z.RefinementCtx,
+): void {
+ const violation = findStrictSchemaViolation(schema);
+ if (violation) {
+ ctx.addIssue({ code: "custom", message: violation });
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts
index 5cedd0537..bc77de7c4 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts
@@ -1,4 +1,13 @@
+import { vi } from "vitest";
+
+vi.mock("../transformers/llmExtract", () => ({
+ generateCompletions: vi.fn(),
+ generateSchemaFromPrompt: vi.fn(),
+}));
+
+import type { Mock } from "vitest";
import { extractData } from "./extractSmartScrape";
+import { generateCompletions } from "../transformers/llmExtract";
import { JsonExtractionContentTooLargeError } from "../error";
import { CostTracking } from "../../../lib/cost-tracking";
@@ -31,4 +40,107 @@ describe("extractData", () => {
}),
).rejects.toBeInstanceOf(JsonExtractionContentTooLargeError);
});
+
+ describe("SmartScrape schema wrapping", () => {
+ function run(schema: any, useAgent: boolean) {
+ return extractData({
+ extractOptions: {
+ logger: noopLogger,
+ options: { schema },
+ markdown: "page",
+ costTrackingOptions: {
+ costTracking: new CostTracking(),
+ metadata: {},
+ },
+ metadata: { teamId: "test-team" },
+ } as any,
+ urls: ["https://example.com"],
+ useAgent,
+ metadata: { teamId: "test-team" },
+ });
+ }
+
+ function sentSchema() {
+ const calls = (generateCompletions as Mock).mock.calls;
+ return calls[calls.length - 1][0].options.schema;
+ }
+
+ function mockExtract(extract: any) {
+ (generateCompletions as Mock).mockResolvedValueOnce({
+ extract,
+ warning: undefined,
+ totalUsage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },
+ });
+ }
+
+ it("sends the user's schema as-is when the agent is off", async () => {
+ const schema = {
+ judgments: { type: "array", items: { type: "string" } },
+ };
+ mockExtract({ judgments: ["a"] });
+
+ const result = await run(schema, false);
+
+ expect(sentSchema()).toEqual(schema);
+ expect(result.extractedDataArray).toEqual([{ judgments: ["a"] }]);
+ });
+
+ it("extracts without a schema when none was given", async () => {
+ mockExtract({ anything: 1 });
+
+ const result = await run(undefined, false);
+
+ expect(sentSchema()).toBeUndefined();
+ expect(result.extractedDataArray).toEqual([{ anything: 1 }]);
+ });
+
+ it("returns root array results as the array itself", async () => {
+ mockExtract({ items: [{ name: "a" }, { name: "b" }] });
+
+ const result = await run(
+ {
+ type: "Array",
+ items: { type: "object", properties: { name: { type: "string" } } },
+ },
+ false,
+ );
+
+ expect(result.extractedDataArray).toEqual([
+ [{ name: "a" }, { name: "b" }],
+ ]);
+ });
+
+ it("still wraps the schema for the agent", async () => {
+ const schema = {
+ type: "object",
+ properties: { title: { type: "string" } },
+ };
+ mockExtract({
+ extractedData: { title: "t" },
+ shouldUseSmartscrape: false,
+ });
+
+ const result = await run(schema, true);
+
+ expect(sentSchema().properties.extractedData).toEqual(schema);
+ expect(sentSchema().required).toContain("extractedData");
+ expect(result.extractedDataArray).toEqual([{ title: "t" }]);
+ });
+
+ it("turns a bare property map into an object schema before wrapping", async () => {
+ mockExtract({
+ extractedData: { title: "t" },
+ shouldUseSmartscrape: false,
+ });
+
+ await run({ title: { type: "string" } }, true);
+
+ expect(sentSchema().properties.extractedData).toEqual({
+ type: "object",
+ properties: { title: { type: "string" } },
+ required: ["title"],
+ additionalProperties: false,
+ });
+ });
+ });
});
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 45424831a..9f902b3a2 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -19,6 +19,7 @@ import {
CostTracking,
} from "../../../lib/cost-tracking";
import { JsonExtractionContentTooLargeError } from "../error";
+import { toRootSchema, typeIncludes } from "../../../lib/openai-strict-schema";
// ~2MB of markdown, well past typical page sizes -- caps worst-case JSON extraction cost/latency.
const MAX_JSON_EXTRACTION_MARKDOWN_CHARS = 2_000_000;
@@ -244,6 +245,22 @@ const resolveRefs = (
return resolved;
};
+// generateCompletions nests a root array schema under an "items" property
+// (structured outputs need an object at the root); hand back the array itself,
+// as the SmartScrape wrapper's extractedData property always did.
+function unwrapRootArray(schema: any, extract: any): any {
+ if (
+ typeIncludes(toRootSchema(schema)?.type, "array") &&
+ extract &&
+ typeof extract === "object" &&
+ !Array.isArray(extract) &&
+ "items" in extract
+ ) {
+ return extract.items;
+ }
+ return extract;
+}
+
export async function extractData({
extractOptions,
urls,
@@ -355,7 +372,16 @@ export async function extractData({
}
}
- const { schemaToUse } = prepareSmartScrapeSchema(schema, logger, isSingleUrl);
+ // The SmartScrape fields only matter when the agent can act on them.
+ // Without it, the user's schema goes to generateCompletions as-is (with no
+ // schema at all, the wrapper would require an extractedData property it
+ // doesn't have). Nested under the wrapper, a bare property map is turned
+ // into an object schema first, as generateCompletions does at the root.
+ const wrapForSmartScrape = useAgent && !!schema;
+ const schemaToUse = wrapForSmartScrape
+ ? prepareSmartScrapeSchema(toRootSchema(schema), logger, isSingleUrl)
+ .schemaToUse
+ : schema;
const extractOptionsNewSchema = {
...extractOptions,
options: { ...extractOptions.options, schema: schemaToUse },
@@ -424,7 +450,9 @@ export async function extractData({
warning = `JSON extraction failed: ${reason.slice(0, 300)}`;
}
- let extractedData = extract?.extractedData;
+ let extractedData = wrapForSmartScrape
+ ? extract?.extractedData
+ : unwrapRootArray(schema, extract);
// console.log("shouldUseSmartscrape", extract?.shouldUseSmartscrape);
// console.log("smartscrape_reasoning", extract?.smartscrape_reasoning);
@@ -438,7 +466,7 @@ export async function extractData({
providedExtractId: extractId,
});
- if (useAgent && extract?.shouldUseSmartscrape) {
+ if (wrapForSmartScrape && extract?.shouldUseSmartscrape) {
// technically this should be checked upstream but might as well add another guard - Mogery
if (extractOptions.zeroDataRetention) {
throw new Error(
diff --git a/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts
new file mode 100644
index 000000000..00ff91858
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts
@@ -0,0 +1,165 @@
+import { vi } from "vitest";
+
+vi.mock("ai", async importOriginal => {
+ const actual = await importOriginal<typeof import("ai")>();
+ return {
+ ...actual,
+ generateObject: vi.fn(async () => ({
+ object: { ok: true },
+ usage: { inputTokens: 1, outputTokens: 1 },
+ })),
+ };
+});
+
+import { generateObject } from "ai";
+import { encoding_for_model } from "@dqbd/tiktoken";
+import type { Mock } from "vitest";
+import { generateCompletions } from "./llmExtract";
+import { CostTracking } from "../../../lib/cost-tracking";
+import { modelPrices } from "../../../lib/extract/usage/model-prices";
+
+const noopLogger = {
+ warn: () => {},
+ info: () => {},
+ error: () => {},
+ debug: () => {},
+ child: () => noopLogger,
+} as any;
+
+function run(markdown: string, modelId: string, schema?: any) {
+ return generateCompletions({
+ logger: noopLogger,
+ options: { schema },
+ markdown,
+ model: { modelId } as any,
+ retryModel: { modelId } as any,
+ costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },
+ metadata: { teamId: "test-team" },
+ });
+}
+
+function lastCall() {
+ const calls = (generateObject as Mock).mock.calls;
+ expect(calls).toHaveLength(1);
+ return calls[0][0];
+}
+
+function countTokens(text: string): number {
+ const encoder = encoding_for_model("gpt-4o-mini");
+ try {
+ return encoder.encode(text).length;
+ } finally {
+ encoder.free();
+ }
+}
+
+beforeEach(() => {
+ (generateObject as Mock).mockClear();
+});
+
+describe("generateCompletions content trimming", () => {
+ // ~3.3 characters per token like real pages: ~400k tokens, three times
+ // gpt-4o-mini's window.
+ const hugeMarkdown =
+ "The quick brown fox jumps over the lazy dog. [link](https://example.com/a/b?c=1) 12345 ".repeat(
+ 15_000,
+ );
+
+ it("trims content that would overflow the model's context window", async () => {
+ const result = await run(hugeMarkdown, "gpt-4o-mini");
+
+ expect(result.warning).toContain(
+ "the input has been automatically trimmed",
+ );
+ // The content is trimmed to exactly 80% of the window; the rest of the
+ // prompt is a short instruction.
+ const budget = Math.floor(
+ modelPrices["gpt-4o-mini"].max_input_tokens * 0.8,
+ );
+ const promptTokens = countTokens(lastCall().prompt);
+ expect(promptTokens).toBeGreaterThan(budget);
+ expect(promptTokens).toBeLessThanOrEqual(budget + 100);
+ });
+
+ it("leaves content that fits untouched", async () => {
+ const markdown = "a small page";
+ const result = await run(markdown, "gpt-4o-mini");
+
+ expect(result.warning).toBeUndefined();
+ expect(lastCall().prompt).toContain(markdown);
+ });
+
+ it("does not trim for models without known limits", async () => {
+ const result = await run(hugeMarkdown, "some-unlisted-model");
+
+ expect(result.warning).toBeUndefined();
+ expect(lastCall().prompt).toContain(hugeMarkdown);
+ });
+});
+
+describe("generateCompletions schema normalization", () => {
+ it("turns a bare property map into an object schema, skipping $schema", async () => {
+ await run("page", "gpt-4o-mini", {
+ $schema: "http://json-schema.org/draft-07/schema#",
+ judgments: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: { citation: { type: "String" } },
+ },
+ },
+ });
+
+ expect(lastCall().schema.jsonSchema).toEqual({
+ type: "object",
+ properties: {
+ judgments: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: { citation: { type: "string" } },
+ required: ["citation"],
+ additionalProperties: false,
+ },
+ },
+ },
+ required: ["judgments"],
+ additionalProperties: false,
+ });
+ });
+
+ it("wraps nullable root arrays in an object", async () => {
+ await run("page", "gpt-4o-mini", {
+ type: ["array", "null"],
+ items: { type: "string" },
+ });
+
+ expect(lastCall().schema.jsonSchema).toEqual({
+ type: "object",
+ properties: {
+ items: { type: ["array", "null"], items: { type: "string" } },
+ },
+ required: ["items"],
+ additionalProperties: false,
+ });
+ });
+
+ it("closes nullable objects so strict mode accepts them", async () => {
+ await run("page", "gpt-4o-mini", {
+ type: "object",
+ properties: {
+ address: {
+ type: ["object", "null"],
+ properties: { city: { type: "string" } },
+ },
+ },
+ });
+
+ expect(lastCall().schema.jsonSchema.properties.address).toEqual({
+ type: ["object", "null"],
+ properties: { city: { type: "string" } },
+ required: ["city"],
+ additionalProperties: false,
+ });
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index ee63de2be..08aa15d22 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -22,6 +22,11 @@ import { z } from "zod";
import fs from "fs/promises";
import Ajv from "ajv";
import { extractData } from "../lib/extractSmartScrape";
+import {
+ findStrictSchemaViolation,
+ toRootSchema,
+ typeIncludes,
+} from "../../../lib/openai-strict-schema";
import { CostTracking } from "../../../lib/cost-tracking";
import { isAgentExtractModelValid } from "../../../controllers/v1/types";
import { hasFormatOfType } from "../../../lib/format-utils";
@@ -131,7 +136,7 @@ function normalizeSchema(x: any): any {
x.not = normalizeSchema(x.not);
}
- if (x && x.type === "object") {
+ if (x && typeIncludes(x.type, "object")) {
return {
...x,
properties: Object.fromEntries(
@@ -143,7 +148,7 @@ function normalizeSchema(x: any): any {
required: Object.keys(x.properties || {}),
additionalProperties: false,
};
- } else if (x && x.type === "array") {
+ } else if (x && typeIncludes(x.type, "array")) {
return {
...x,
items: normalizeSchema(x.items),
@@ -342,6 +347,20 @@ export async function generateCompletions({
throw new Error("document.markdown is undefined -- this is unexpected");
}
+ // Keep the content inside the model's context window, leaving the rest for
+ // the prompt, schema and output. Models without known limits are sent the
+ // content as-is. A BPE token is at least one byte, so content that fits in
+ // bytes skips the (synchronous) tokenizer entirely.
+ const maxInputTokens = modelPrices[modelId]?.max_input_tokens;
+ if (markdown && maxInputTokens) {
+ const maxContentTokens = Math.floor(maxInputTokens * 0.8);
+ if (Buffer.byteLength(markdown, "utf8") > maxContentTokens) {
+ const trimmed = trimToTokenLimit(markdown, maxContentTokens, modelId);
+ markdown = trimmed.text;
+ warning = trimmed.warning;
+ }
+ }
+
try {
const prompt =
options.prompt !== undefined
@@ -565,29 +584,21 @@ export async function generateCompletions({
if (schema && !(schema instanceof z.ZodType)) {
// let schema = options.schema;
if (schema) {
- schema = removeDefaultProperty(schema);
+ schema = toRootSchema(removeDefaultProperty(schema));
}
- if (schema && schema.type === "array") {
+ // Structured outputs need a (non-nullable) object at the root.
+ if (schema && typeIncludes(schema.type, "array")) {
schema = {
type: "object",
properties: {
- items: options.schema,
+ items: schema,
},
required: ["items"],
additionalProperties: false,
};
- } else if (schema && typeof schema === "object" && !schema.type) {
- schema = {
- type: "object",
- properties: Object.fromEntries(
- Object.entries(schema).map(([key, value]) => {
- return [key, removeDefaultProperty(value)];
- }),
- ),
- required: Object.keys(schema),
- additionalProperties: false,
- };
+ } else if (schema && typeIncludes(schema.type, "object")) {
+ schema = { ...schema, type: "object" };
}
schema = normalizeSchema(schema);
@@ -1469,6 +1480,9 @@ export async function generateSchemaFromPrompt(
const retryModel = getModel("gpt-4.1-mini", "openai");
const temperatures = [0, 0.1, 0.3]; // Different temperatures to try
let lastError: Error | null = null;
+ // The last schema strict mode would reject, used only if no attempt
+ // produces a supported one (extraction then fails with a warning as before).
+ let lastUnsupportedSchema: any = undefined;
for (const temp of temperatures) {
try {
@@ -1489,16 +1503,18 @@ Consider:
Valid JSON schema, has to be simple. No crazy properties. OpenAI has to support it.
Supported types
-The following types are supported for Structured Outputs:
+The following values of "type" are supported for Structured Outputs, written exactly like this (lowercase):
+
+string
+number
+boolean
+integer
+object
+array
-String
-Number
-Boolean
-Integer
-Object
-Array
-Enum
-anyOf
+Use "enum" as a keyword next to "type": "string" to restrict values; it is not a type. Use "anyOf" for alternatives.
+Every array must define "items" as a single schema object. Every property must be a schema object with a "type".
+allOf, oneOf, not, and if/then/else are not supported.
Formats are not supported. Min/max are not supported. Anything beyond the above is not supported. Keep it simple with types and descriptions.
Optionals are not supported.
@@ -1524,7 +1540,15 @@ Return a valid JSON schema object with properties that would capture the informa
zeroDataRetention,
});
- return { extract };
+ const violation = findStrictSchemaViolation(extract);
+ if (violation === null) {
+ return { extract };
+ }
+ logger.warn("Generated schema is not supported by structured outputs", {
+ violation,
+ temperature: temp,
+ });
+ lastUnsupportedSchema = extract;
} catch (error) {
lastError = error as Error;
logger.warn(`Failed attempt with temperature ${temp}: ${error.message}`);
@@ -1532,6 +1556,10 @@ Return a valid JSON schema object with properties that would capture the informa
}
}
+ if (lastUnsupportedSchema !== undefined) {
+ return { extract: lastUnsupportedSchema };
+ }
+
// If we get here, all attempts failed
throw new Error(
`Failed to generate schema after all attempts. Last error: ${lastError?.message}`,