Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(json): stop JSON extraction failing on schema wrapping, oversized pages and unsupported schemas (#4807)
Gergő Móricz2c88cbb041cd
diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.tsindex 0e98bf04d..89dedbb10 100644--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts@@ -985,7 +985,7 @@ describe("V1 Types Validation", () => {schema: {type: "object",properties: {- changes: { type: "array" },+ changes: { type: "array", items: { type: "string" } },},},modes: ["json"],diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex dbd201e44..b787a2c89 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -1685,6 +1685,34 @@ describe("Scrape tests", () => {);});+ it.concurrent(+ "rejects JSON schemas that structured outputs cannot express",+ async () => {+ const raw = await scrapeRaw(+ {+ url: base,+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: { events: { type: "array" } },+ },+ },+ ],+ },+ identity,+ );++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe(+ 'Invalid JSON schema at "properties.events": arrays must define "items".',+ );+ },+ scrapeTimeout,+ );+describeIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))("JSON format",() => {@@ -1736,6 +1764,54 @@ describe("Scrape tests", () => {},scrapeTimeout,);++ it.concurrent(+ "works with a bare map of property names to schemas",+ async () => {+ const response = await scrape(+ {+ url: base,+ formats: [+ {+ type: "json",+ schema: {+ company_name: { type: "string" },+ is_open_source: { type: "boolean" },+ },+ },+ ],+ },+ identity,+ );++ expect(response.warning ?? "").not.toContain(+ "JSON extraction failed",+ );+ expect(typeof response.json.company_name).toBe("string");+ expect(typeof response.json.is_open_source).toBe("boolean");+ },+ scrapeTimeout,+ );++ it.concurrent(+ "works without a schema or prompt",+ async () => {+ const response = await scrape(+ {+ url: base,+ formats: [{ type: "json" }],+ },+ identity,+ );++ expect(response.warning ?? "").not.toContain(+ "JSON extraction failed",+ );+ expect(response.json).not.toBeNull();+ expect(typeof response.json).toBe("object");+ },+ scrapeTimeout,+ );},);diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex 36dff1e21..48bc79bfe 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -149,6 +149,72 @@ describe("V2 Types Validation", () => {expect(result.timeout).toBe(60000); // Should be transformed from 30000});+ it("should reject JSON schemas structured outputs cannot express", () => {+ const input: ScrapeRequestInput = {+ url: "https://example.com",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: { events: { type: "array" } },+ },+ },+ ],+ };++ const result = scrapeRequestSchema.safeParse(input);+ expect(result.success).toBe(false);+ expect(result.error?.issues.map(issue => issue.message)).toContain(+ 'Invalid JSON schema at "properties.events": arrays must define "items".',+ );+ });++ it("should accept JSON schemas with capitalized types and oneOf", () => {+ const input: ScrapeRequestInput = {+ url: "https://example.com",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: {+ name: { type: "String" },+ slot: { oneOf: [{ type: "string" }, { type: "number" }] },+ },+ },+ },+ ],+ };++ expect(() => scrapeRequestSchema.parse(input)).not.toThrow();+ });++ it("should reject unsupported changeTracking schemas", () => {+ const input: ScrapeRequestInput = {+ url: "https://example.com",+ formats: [+ "markdown",+ {+ type: "changeTracking",+ modes: ["json"],+ schema: {+ type: "object",+ properties: {+ price: { type: "object", properties: {}, allOf: [] },+ },+ },+ },+ ],+ };++ const result = scrapeRequestSchema.safeParse(input);+ expect(result.success).toBe(false);+ expect(result.error?.issues.map(issue => issue.message)).toContain(+ 'Invalid JSON schema at "properties.price": "allOf" is not supported for JSON extraction.',+ );+ });+it("should accept query format without markdown", () => {const input: ScrapeRequestInput = {url: "https://example.com",@@ -245,7 +311,7 @@ describe("V2 Types Validation", () => {schema: {type: "object",properties: {- changes: { type: "array" },+ changes: { type: "array", items: { type: "string" } },},},modes: ["json"],@@ -805,6 +871,21 @@ describe("V2 Types Validation", () => {expect(result.urls).toHaveLength(10);});+ it("should reject JSON schemas structured outputs cannot express", () => {+ const result = extractRequestSchema.safeParse({+ urls: ["https://example.com"],+ schema: {+ type: "object",+ properties: { events: { type: "array" } },+ },+ });++ expect(result.success).toBe(false);+ expect(result.error?.issues.map(issue => issue.message)).toContain(+ 'Invalid JSON schema at "properties.events": arrays must define "items".',+ );+ });+it("should reject invalid JSON schema", () => {const input: ExtractRequestInput = {urls: ["https://example.com"],@@ -1984,7 +2065,7 @@ describe("V2 Types Validation", () => {schema: {type: "object",properties: {- changes: { type: "array" },+ changes: { type: "array", items: { type: "string" } },},},},diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex ec34c63eb..9d6386358 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -5,6 +5,7 @@ import { protocolIncluded, checkUrl } from "../../lib/validateUrl";import { hasReachableHost } from "../../lib/url-utils";import { countries } from "../../lib/validate-country";import { addPathRegexIssues, pathPatternsSchema } from "../../lib/crawl-regex";+import { addStrictSchemaIssue } from "../../lib/openai-strict-schema";import type { PdfPageBlocks } from "../../scraper/scrapeURL/engines/pdf/types";import {ExtractorOptions,@@ -219,7 +220,8 @@ export const extractOptions = z.transform(val => normalizeSchemaForOpenAI(val)).refine(val => validateSchemaForOpenAI(val), {message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ })+ .superRefine(addStrictSchemaIssue),systemPrompt: z.string().max(10000).prefault(""),prompt: z.string().max(10000).optional(),temperature: z.number().optional(),@@ -254,7 +256,8 @@ const extractOptionsWithAgent = z.transform(val => normalizeSchemaForOpenAI(val)).refine(val => validateSchemaForOpenAI(val), {message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ })+ .superRefine(addStrictSchemaIssue),systemPrompt: z.string().max(10000).prefault(""),prompt: z.string().max(10000).optional(),temperature: z.number().optional(),@@ -509,7 +512,8 @@ const baseScrapeOptions = z.strictObject({.transform(val => normalizeSchemaForOpenAI(val)).refine(val => validateSchemaForOpenAI(val), {message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ })+ .superRefine(addStrictSchemaIssue),modes: z.enum(["json", "git-diff"]).array().optional().prefault([]),tag: z.string().or(z.null()).prefault(null),})@@ -765,7 +769,8 @@ const extractV1Options = z.transform(val => normalizeSchemaForOpenAI(val)).refine(val => validateSchemaForOpenAI(val), {message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ })+ .superRefine(addStrictSchemaIssue),limit: z.int().positive().finite().optional(),ignoreSitemap: z.boolean().prefault(false),includeSubdomains: z.boolean().prefault(true),diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 515a5ed7a..18bf6ea39 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -8,6 +8,7 @@ import { hasReachableHost } from "../../lib/url-utils";import { countries } from "../../lib/validate-country";import { includesFormat } from "../../lib/format-utils";import { addPathRegexIssues, pathPatternsSchema } from "../../lib/crawl-regex";+import { addStrictSchemaIssue } from "../../lib/openai-strict-schema";import {ExtractorOptions,PageOptions,@@ -359,7 +360,8 @@ const jsonFormatWithOptions = z.strictObject({.transform(val => normalizeSchemaForOpenAI(val)).refine(val => validateSchemaForOpenAI(val), {message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ })+ .superRefine(addStrictSchemaIssue),prompt: z.string().max(10000).optional(),checkPromptInjection: z.boolean().optional(),});@@ -394,7 +396,8 @@ const changeTrackingFormatWithOptions = z.strictObject({.transform(val => normalizeSchemaForOpenAI(val)).refine(val => validateSchemaForOpenAI(val), {message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ })+ .superRefine(addStrictSchemaIssue),modes: z.enum(["json", "git-diff"]).array().optional().prefault([]),tag: z.string().or(z.null()).prefault(null),});@@ -1004,7 +1007,8 @@ const extractOptions = z.transform(val => normalizeSchemaForOpenAI(val)).refine(val => validateSchemaForOpenAI(val), {message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ })+ .superRefine(addStrictSchemaIssue),limit: z.int().positive().finite().optional(),ignoreSitemap: z.boolean().prefault(false),includeSubdomains: z.boolean().prefault(true),diff --git a/apps/api/src/lib/openai-strict-schema.test.ts b/apps/api/src/lib/openai-strict-schema.test.tsnew file mode 100644index 000000000..86ce879b2--- /dev/null+++ b/apps/api/src/lib/openai-strict-schema.test.ts@@ -0,0 +1,450 @@+import { z } from "zod";+import {+ addStrictSchemaIssue,+ findStrictSchemaViolation,+ normalizeSchemaKeywords,+ toRootSchema,+} from "./openai-strict-schema";++describe("normalizeSchemaKeywords", () => {+ it("lowercases type names, including nested and nullable ones", () => {+ const result = normalizeSchemaKeywords({+ type: "Object",+ properties: {+ name: { type: "String" },+ count: { type: ["Integer", "null"] },+ tags: { type: "Array", items: { type: "String" } },+ },+ });++ expect(result).toEqual({+ type: "object",+ properties: {+ name: { type: "string" },+ count: { type: ["integer", "null"] },+ tags: { type: "array", items: { type: "string" } },+ },+ });+ });++ it("turns oneOf into anyOf", () => {+ const result = normalizeSchemaKeywords({+ type: "object",+ properties: {+ slot: { oneOf: [{ type: "string" }, { type: "Number" }] },+ },+ });++ expect(result.properties.slot).toEqual({+ anyOf: [{ type: "string" }, { type: "number" }],+ });+ });++ it("infers a missing type from properties or items below the root", () => {+ const result = normalizeSchemaKeywords({+ type: "object",+ properties: {+ identity: { properties: { name: { type: "string" } } },+ list: { items: { type: "string" } },+ },+ });++ expect(result.properties.identity.type).toBe("object");+ expect(result.properties.list.type).toBe("array");+ });++ it("leaves a root map of property names alone, even with an 'items' field", () => {+ const schema = {+ items: { type: "array", items: { type: "string" } },+ total: { type: "number" },+ };++ expect(normalizeSchemaKeywords(schema)).toEqual(schema);+ });++ it("infers object for a typeless root that only uses schema keywords", () => {+ const result = normalizeSchemaKeywords({+ $schema: "http://json-schema.org/draft-07/schema#",+ properties: { name: { type: "string" } },+ required: ["name"],+ });++ expect(result.type).toBe("object");+ });++ it("drops additionalProperties schemas next to properties", () => {+ const result = normalizeSchemaKeywords({+ type: "object",+ properties: { name: { type: "string" } },+ additionalProperties: { type: "array" },+ });++ expect(result.additionalProperties).toBe(false);+ });++ it("does not modify its input", () => {+ const schema = { type: "String" };+ normalizeSchemaKeywords(schema);+ expect(schema.type).toBe("String");+ });+});++describe("toRootSchema", () => {+ it("turns a bare map into an object schema, skipping $schema", () => {+ expect(+ toRootSchema({+ $schema: "http://json-schema.org/draft-07/schema#",+ name: { type: "String" },+ }),+ ).toEqual({+ type: "object",+ properties: { name: { type: "string" } },+ required: ["name"],+ additionalProperties: false,+ });+ });++ it("keeps $-named fields other than annotations, and moves $defs to the root", () => {+ expect(+ toRootSchema({+ $schema: "http://json-schema.org/draft-07/schema#",+ $metadata: { type: "string" },+ item: { $ref: "#/$defs/item" },+ $defs: { item: { type: "object", properties: {} } },+ }),+ ).toEqual({+ type: "object",+ properties: {+ $metadata: { type: "string" },+ item: { $ref: "#/$defs/item" },+ },+ required: ["$metadata", "item"],+ additionalProperties: false,+ $defs: { item: { type: "object", properties: {} } },+ });+ });++ it("treats a field named 'properties' as a field, not as a schema keyword", () => {+ const result = toRootSchema({+ title: { type: "string" },+ properties: {+ type: "array",+ items: { type: "object", properties: { price: { type: "number" } } },+ },+ });++ expect(result.type).toBe("object");+ expect(Object.keys(result.properties)).toEqual(["title", "properties"]);+ expect(result.properties.properties.type).toBe("array");+ });++ it("keeps a typeless root schema a schema", () => {+ const result = toRootSchema({+ title: "Listing",+ properties: { price: { type: "number" } },+ required: ["price"],+ });++ expect(result).toEqual({+ title: "Listing",+ type: "object",+ properties: { price: { type: "number" } },+ required: ["price"],+ });+ });+});++describe("findStrictSchemaViolation", () => {+ it("accepts a plain object schema", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: {+ title: { type: "string" },+ tags: { type: "array", items: { type: "string" } },+ status: { type: "string", enum: ["open", "closed"] },+ price: { anyOf: [{ type: "number" }, { type: "null" }] },+ },+ required: ["title"],+ }),+ ).toBeNull();+ });++ it("accepts undefined (no schema)", () => {+ expect(findStrictSchemaViolation(undefined)).toBeNull();+ });++ it("accepts what normalizeSchemaKeywords fixes", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: {+ name: { type: "String" },+ slot: { oneOf: [{ type: "string" }, { type: "integer" }] },+ identity: { properties: { value: { type: "string" } } },+ },+ }),+ ).toBeNull();+ });++ it("accepts a root map of property names to schemas", () => {+ expect(+ findStrictSchemaViolation({+ $schema: "http://json-schema.org/draft-07/schema#",+ judgments: {+ type: "array",+ items: {+ type: "object",+ properties: { citation: { type: "string" } },+ },+ },+ }),+ ).toBeNull();+ });++ it("accepts a bare map with a field named 'properties'", () => {+ expect(+ findStrictSchemaViolation({+ title: { type: "string" },+ properties: { type: "array", items: { type: "string" } },+ }),+ ).toBeNull();+ });++ it("accepts nullable root arrays", () => {+ expect(+ findStrictSchemaViolation({+ type: ["array", "null"],+ items: { type: "string" },+ }),+ ).toBeNull();+ });++ it("rejects roots that are neither objects nor arrays", () => {+ expect(findStrictSchemaViolation({ type: "string" })).toBe(+ 'Invalid JSON schema: the root must be an object or array schema, got type "string".',+ );+ expect(+ findStrictSchemaViolation({+ anyOf: [{ type: "object", properties: {} }, { type: "null" }],+ }),+ ).toBe(+ 'Invalid JSON schema: the root must be an object or array schema, not "anyOf".',+ );+ });++ it("checks definitions behind a root $ref", () => {+ expect(+ findStrictSchemaViolation({+ $ref: "#/$defs/listing",+ $defs: {+ listing: {+ type: "object",+ properties: { photos: { type: "array" } },+ },+ },+ }),+ ).toBe(+ 'Invalid JSON schema at "$defs.listing.properties.photos": arrays must define "items".',+ );+ });++ it("checks keywords next to a $ref", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: { node: { $ref: "#/$defs/node", allOf: [] } },+ $defs: { node: { type: "object", properties: {} } },+ }),+ ).toBe(+ 'Invalid JSON schema at "properties.node": "allOf" is not supported for JSON extraction.',+ );+ });++ it("accepts recursive schemas using $ref", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: { root: { $ref: "#/$defs/node" } },+ $defs: {+ node: {+ type: "object",+ properties: {+ children: { type: "array", items: { $ref: "#/$defs/node" } },+ },+ },+ },+ }),+ ).toBeNull();+ });++ it("rejects arrays without items, naming the path", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: {+ centers: {+ type: "array",+ items: {+ type: "object",+ properties: { professionals: { type: "array" } },+ },+ },+ },+ }),+ ).toBe(+ 'Invalid JSON schema at "properties.centers.items.properties.professionals": arrays must define "items".',+ );+ });++ it("rejects tuple-style items", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: {+ row: {+ type: "array",+ items: [{ type: "string" }, { type: "number" }],+ },+ },+ }),+ ).toContain('"items" must be a single schema object');+ });++ it.each([+ "allOf",+ "not",+ "if",+ "then",+ "else",+ "dependentRequired",+ "dependentSchemas",+ ])("rejects %s", keyword => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: {+ data: { type: "object", properties: {}, [keyword]: [] },+ },+ }),+ ).toBe(+ `Invalid JSON schema at "properties.data": "${keyword}" is not supported for JSON extraction.`,+ );+ });++ it("rejects oneOf when anyOf is also present", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: {+ slot: {+ anyOf: [{ type: "string" }],+ oneOf: [{ type: "number" }],+ },+ },+ }),+ ).toBe(+ 'Invalid JSON schema at "properties.slot": "oneOf" is not supported for JSON extraction.',+ );+ });++ it("rejects non-object properties and non-array anyOf", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: { data: { type: "object", properties: ["a", "b"] } },+ }),+ ).toBe(+ 'Invalid JSON schema at "properties.data": "properties" must be an object, got ["a","b"].',+ );+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: { slot: { anyOf: { type: "string" } } },+ }),+ ).toBe(+ 'Invalid JSON schema at "properties.slot": "anyOf" must be an array, got {"type":"string"}.',+ );+ });++ it("ignores additionalProperties schemas next to properties", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: { name: { type: "string" } },+ additionalProperties: { type: "array" },+ }),+ ).toBeNull();+ });++ it("rejects typed dictionaries", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: {+ prices: { type: "object", additionalProperties: { type: "number" } },+ },+ }),+ ).toBe(+ 'Invalid JSON schema at "properties.prices": objects must list their "properties"; a dictionary described only by "additionalProperties" is not supported for JSON extraction.',+ );+ });++ it("rejects type names that are not JSON Schema types", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: { level: { type: "enum", enum: ["a", "b"] } },+ }),+ ).toContain('"type" must be one of');+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: { title: { type: { type: "string" } } },+ }),+ ).toContain('"type" must be one of');+ });++ it("rejects shorthand property values", () => {+ expect(+ findStrictSchemaViolation({ status: "string", price: "string" }),+ ).toBe(+ 'Invalid JSON schema at "properties.status": expected a schema object, got "string".',+ );+ });++ it("rejects nested schemas without a type", () => {+ expect(+ findStrictSchemaViolation({+ type: "object",+ properties: { metadata: {} },+ }),+ ).toBe('Invalid JSON schema at "properties.metadata": missing "type".');+ });+});++describe("addStrictSchemaIssue", () => {+ const schemaField = z.any().optional().superRefine(addStrictSchemaIssue);++ it("adds a zod issue for an unsupported schema", () => {+ const result = schemaField.safeParse({+ type: "object",+ properties: { events: { type: "array" } },+ });++ expect(result.success).toBe(false);+ expect(result.error?.issues[0].message).toBe(+ 'Invalid JSON schema at "properties.events": arrays must define "items".',+ );+ });++ it("passes supported and missing schemas", () => {+ expect(schemaField.safeParse(undefined).success).toBe(true);+ expect(+ schemaField.safeParse({+ type: "object",+ properties: { title: { type: "string" } },+ }).success,+ ).toBe(true);+ });+});diff --git a/apps/api/src/lib/openai-strict-schema.ts b/apps/api/src/lib/openai-strict-schema.tsnew file mode 100644index 000000000..cfebfa714--- /dev/null+++ b/apps/api/src/lib/openai-strict-schema.ts@@ -0,0 +1,315 @@+import { z } from "zod";++// OpenAI structured outputs (strict mode) accept only a subset of JSON Schema.+// normalizeSchemaKeywords fixes the mechanical mistakes we see in user-written+// and LLM-generated schemas; findStrictSchemaViolation names the constructs+// strict mode rejects outright, so they can be refused before any scrape runs.++const JSON_SCHEMA_TYPES = new Set([+ "string",+ "number",+ "integer",+ "boolean",+ "object",+ "array",+ "null",+]);++// Rejected by strict mode wherever they appear.+const UNSUPPORTED_KEYWORDS = [+ "allOf",+ "oneOf",+ "not",+ "if",+ "then",+ "else",+ "dependentRequired",+ "dependentSchemas",+];++type Schema = Record<string, any>;++function isPlainObject(x: unknown): x is Schema {+ return typeof x === "object" && x !== null && !Array.isArray(x);+}++function mapValues(obj: Schema, fn: (v: any) => any): Schema {+ return Object.fromEntries(Object.entries(obj).map(([k, v]) => [k, fn(v)]));+}++function lowercaseType(type: unknown): unknown {+ if (typeof type === "string") return type.toLowerCase();+ if (Array.isArray(type)) {+ return type.map(t => (typeof t === "string" ? t.toLowerCase() : t));+ }+ return type;+}++export function typeIncludes(type: unknown, name: string): boolean {+ return type === name || (Array.isArray(type) && type.includes(name));+}++// Whether a typeless root is a schema rather than a bare map of property names+// to schemas (which JSON extraction also accepts at the root): every key is a+// schema keyword holding the kind of value that keyword takes, and+// "properties" maps names to schemas instead of being a schema itself (a+// field that happens to be called "properties").+function isTypelessRootSchema(node: Schema): boolean {+ if (!isPlainObject(node.properties)) return false;+ if (typeof node.properties.type === "string") return false;+ return Object.entries(node).every(([key, value]) => {+ switch (key) {+ case "$schema":+ case "$id":+ case "$comment":+ case "title":+ case "description":+ return typeof value === "string";+ case "required":+ return Array.isArray(value);+ case "additionalProperties":+ return typeof value === "boolean";+ case "properties":+ case "$defs":+ case "definitions":+ return isPlainObject(value);+ default:+ return false;+ }+ });+}++function isBareRootMap(node: unknown): node is Schema {+ return (+ isPlainObject(node) &&+ node.type === undefined &&+ node.anyOf === undefined &&+ node.oneOf === undefined &&+ node.$ref === undefined &&+ !isTypelessRootSchema(node)+ );+}++/**+ * Lowercases type names ("String" -> "string"), turns oneOf into anyOf, and+ * fills in a missing type when properties/items make it unambiguous. Returns+ * a new schema; the input is not modified.+ */+export function normalizeSchemaKeywords(schema: any): any {+ return normalizeNode(schema, true);+}++function normalizeNode(node: any, isRoot: boolean): any {+ if (!isPlainObject(node)) return node;+ const out: Schema = { ...node };++ if (out.type !== undefined) {+ out.type = lowercaseType(out.type);+ }++ if (Array.isArray(out.oneOf) && out.anyOf === undefined) {+ out.anyOf = out.oneOf;+ delete out.oneOf;+ }++ if (out.type === undefined) {+ if (isRoot ? isTypelessRootSchema(out) : isPlainObject(out.properties)) {+ out.type = "object";+ } else if (!isRoot && isPlainObject(out.items)) {+ out.type = "array";+ }+ }++ if (isPlainObject(out.properties)) {+ out.properties = mapValues(out.properties, v => normalizeNode(v, false));+ }+ if (isPlainObject(out.items)) {+ out.items = normalizeNode(out.items, false);+ }+ if (Array.isArray(out.anyOf)) {+ out.anyOf = out.anyOf.map(v => normalizeNode(v, false));+ }+ // Strict mode closes every object (additionalProperties: false), so keys+ // beyond "properties" never come back; a schema for them is dropped here+ // rather than validated. Without "properties", it's a typed dictionary,+ // which findStrictSchemaViolation reports.+ if (isPlainObject(out.additionalProperties)) {+ out.additionalProperties = isPlainObject(out.properties)+ ? false+ : normalizeNode(out.additionalProperties, false);+ }+ for (const defsKey of ["$defs", "definitions"]) {+ if (isPlainObject(out[defsKey])) {+ out[defsKey] = mapValues(out[defsKey], v => normalizeNode(v, false));+ }+ }++ return out;+}++// Annotations that can sit next to the fields of a bare map. Any other key,+// "$"-prefixed or not, is a property name.+const ROOT_ANNOTATIONS = new Set(["$schema", "$id", "$comment"]);++/**+ * normalizeSchemaKeywords, plus turning a bare map of property names to+ * schemas into an object schema. Its "$defs" (a map of definitions, not a+ * schema) moves to the root of that object schema.+ */+export function toRootSchema(schema: any): any {+ if (!isBareRootMap(schema)) return normalizeSchemaKeywords(schema);+ const { $defs, ...rest } = schema;+ const hasDefs = isPlainObject($defs) && typeof $defs.type !== "string";+ const fields = hasDefs ? rest : schema;+ const entries = Object.entries(fields).filter(+ ([key]) => !ROOT_ANNOTATIONS.has(key),+ );+ return normalizeSchemaKeywords({+ type: "object",+ properties: Object.fromEntries(entries),+ required: entries.map(([key]) => key),+ additionalProperties: false,+ ...(hasDefs ? { $defs } : {}),+ });+}++function describe(value: unknown): string {+ const json = JSON.stringify(value) ?? String(value);+ return json.length > 60 ? json.slice(0, 57) + "..." : json;+}++/**+ * Returns a human-readable description of the first construct in the schema+ * that OpenAI strict structured outputs rejects, or null if none was found.+ * The schema is normalized with normalizeSchemaKeywords first, so anything+ * that function fixes is not reported.+ */+export function findStrictSchemaViolation(schema: any): string | null {+ if (schema === undefined || schema === null) return null;+ const root = toRootSchema(schema);+ if (!isPlainObject(root)) {+ return `Invalid JSON schema: expected an object, got ${describe(root)}.`;+ }++ // Structured outputs need an object at the root; JSON extraction wraps a+ // root array in one, but nothing else.+ if (root.$ref === undefined) {+ if (root.anyOf !== undefined) {+ return 'Invalid JSON schema: the root must be an object or array schema, not "anyOf".';+ }+ if (+ !typeIncludes(root.type, "object") &&+ !typeIncludes(root.type, "array")+ ) {+ return `Invalid JSON schema: the root must be an object or array schema, got type ${describe(root.type)}.`;+ }+ }++ return checkNode(root, "");+}++function checkNode(node: any, path: string): string | null {+ const at = path ? ` at "${path}"` : "";+ const child = (segment: string) => (path ? `${path}.${segment}` : segment);++ if (!isPlainObject(node)) {+ return `Invalid JSON schema${at}: expected a schema object, got ${describe(node)}.`;+ }++ for (const keyword of UNSUPPORTED_KEYWORDS) {+ if (node[keyword] !== undefined) {+ return `Invalid JSON schema${at}: "${keyword}" is not supported for JSON extraction.`;+ }+ }++ // A reference takes its type from the definition it points to, which is+ // checked where it is declared.+ const isReference = node.$ref !== undefined;++ if (+ !isReference &&+ node.type === undefined &&+ node.anyOf === undefined &&+ node.enum === undefined &&+ node.const === undefined+ ) {+ return `Invalid JSON schema${at}: missing "type".`;+ }++ if (node.type !== undefined) {+ const types = Array.isArray(node.type) ? node.type : [node.type];+ const invalid = types.find(+ t => typeof t !== "string" || !JSON_SCHEMA_TYPES.has(t),+ );+ if (types.length === 0 || invalid !== undefined) {+ return `Invalid JSON schema${at}: "type" must be one of ${[...JSON_SCHEMA_TYPES].join(", ")}, got ${describe(node.type)}.`;+ }+ }++ if (+ !isReference &&+ typeIncludes(node.type, "array") &&+ !isPlainObject(node.items)+ ) {+ return node.items === undefined+ ? `Invalid JSON schema${at}: arrays must define "items".`+ : `Invalid JSON schema${at}: "items" must be a single schema object, got ${describe(node.items)}.`;+ }++ if (+ typeIncludes(node.type, "object") &&+ node.properties === undefined &&+ isPlainObject(node.additionalProperties)+ ) {+ return `Invalid JSON schema${at}: objects must list their "properties"; a dictionary described only by "additionalProperties" is not supported for JSON extraction.`;+ }++ if (node.properties !== undefined) {+ if (!isPlainObject(node.properties)) {+ return `Invalid JSON schema${at}: "properties" must be an object, got ${describe(node.properties)}.`;+ }+ for (const [key, value] of Object.entries(node.properties)) {+ const violation = checkNode(value, child(`properties.${key}`));+ if (violation) return violation;+ }+ }++ if (isPlainObject(node.items)) {+ const violation = checkNode(node.items, child("items"));+ if (violation) return violation;+ }++ if (node.anyOf !== undefined) {+ if (!Array.isArray(node.anyOf)) {+ return `Invalid JSON schema${at}: "anyOf" must be an array, got ${describe(node.anyOf)}.`;+ }+ for (let i = 0; i < node.anyOf.length; i++) {+ const violation = checkNode(node.anyOf[i], child(`anyOf.${i}`));+ if (violation) return violation;+ }+ }++ for (const defsKey of ["$defs", "definitions"]) {+ if (isPlainObject(node[defsKey])) {+ for (const [key, value] of Object.entries(node[defsKey])) {+ const violation = checkNode(value, child(`${defsKey}.${key}`));+ if (violation) return violation;+ }+ }+ }++ return null;+}++// Zod refinement for request schemas that JSON extraction sends to OpenAI in+// strict mode: an unsupported schema would otherwise fail mid-scrape and come+// back as a warning on an otherwise successful response.+export function addStrictSchemaIssue(+ schema: unknown,+ ctx: z.RefinementCtx,+): void {+ const violation = findStrictSchemaViolation(schema);+ if (violation) {+ ctx.addIssue({ code: "custom", message: violation });+ }+}diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.tsindex 5cedd0537..bc77de7c4 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.test.ts@@ -1,4 +1,13 @@+import { vi } from "vitest";++vi.mock("../transformers/llmExtract", () => ({+ generateCompletions: vi.fn(),+ generateSchemaFromPrompt: vi.fn(),+}));++import type { Mock } from "vitest";import { extractData } from "./extractSmartScrape";+import { generateCompletions } from "../transformers/llmExtract";import { JsonExtractionContentTooLargeError } from "../error";import { CostTracking } from "../../../lib/cost-tracking";@@ -31,4 +40,107 @@ describe("extractData", () => {}),).rejects.toBeInstanceOf(JsonExtractionContentTooLargeError);});++ describe("SmartScrape schema wrapping", () => {+ function run(schema: any, useAgent: boolean) {+ return extractData({+ extractOptions: {+ logger: noopLogger,+ options: { schema },+ markdown: "page",+ costTrackingOptions: {+ costTracking: new CostTracking(),+ metadata: {},+ },+ metadata: { teamId: "test-team" },+ } as any,+ urls: ["https://example.com"],+ useAgent,+ metadata: { teamId: "test-team" },+ });+ }++ function sentSchema() {+ const calls = (generateCompletions as Mock).mock.calls;+ return calls[calls.length - 1][0].options.schema;+ }++ function mockExtract(extract: any) {+ (generateCompletions as Mock).mockResolvedValueOnce({+ extract,+ warning: undefined,+ totalUsage: { promptTokens: 1, completionTokens: 1, totalTokens: 2 },+ });+ }++ it("sends the user's schema as-is when the agent is off", async () => {+ const schema = {+ judgments: { type: "array", items: { type: "string" } },+ };+ mockExtract({ judgments: ["a"] });++ const result = await run(schema, false);++ expect(sentSchema()).toEqual(schema);+ expect(result.extractedDataArray).toEqual([{ judgments: ["a"] }]);+ });++ it("extracts without a schema when none was given", async () => {+ mockExtract({ anything: 1 });++ const result = await run(undefined, false);++ expect(sentSchema()).toBeUndefined();+ expect(result.extractedDataArray).toEqual([{ anything: 1 }]);+ });++ it("returns root array results as the array itself", async () => {+ mockExtract({ items: [{ name: "a" }, { name: "b" }] });++ const result = await run(+ {+ type: "Array",+ items: { type: "object", properties: { name: { type: "string" } } },+ },+ false,+ );++ expect(result.extractedDataArray).toEqual([+ [{ name: "a" }, { name: "b" }],+ ]);+ });++ it("still wraps the schema for the agent", async () => {+ const schema = {+ type: "object",+ properties: { title: { type: "string" } },+ };+ mockExtract({+ extractedData: { title: "t" },+ shouldUseSmartscrape: false,+ });++ const result = await run(schema, true);++ expect(sentSchema().properties.extractedData).toEqual(schema);+ expect(sentSchema().required).toContain("extractedData");+ expect(result.extractedDataArray).toEqual([{ title: "t" }]);+ });++ it("turns a bare property map into an object schema before wrapping", async () => {+ mockExtract({+ extractedData: { title: "t" },+ shouldUseSmartscrape: false,+ });++ await run({ title: { type: "string" } }, true);++ expect(sentSchema().properties.extractedData).toEqual({+ type: "object",+ properties: { title: { type: "string" } },+ required: ["title"],+ additionalProperties: false,+ });+ });+ });});diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex 45424831a..9f902b3a2 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -19,6 +19,7 @@ import {CostTracking,} from "../../../lib/cost-tracking";import { JsonExtractionContentTooLargeError } from "../error";+import { toRootSchema, typeIncludes } from "../../../lib/openai-strict-schema";// ~2MB of markdown, well past typical page sizes -- caps worst-case JSON extraction cost/latency.const MAX_JSON_EXTRACTION_MARKDOWN_CHARS = 2_000_000;@@ -244,6 +245,22 @@ const resolveRefs = (return resolved;};+// generateCompletions nests a root array schema under an "items" property+// (structured outputs need an object at the root); hand back the array itself,+// as the SmartScrape wrapper's extractedData property always did.+function unwrapRootArray(schema: any, extract: any): any {+ if (+ typeIncludes(toRootSchema(schema)?.type, "array") &&+ extract &&+ typeof extract === "object" &&+ !Array.isArray(extract) &&+ "items" in extract+ ) {+ return extract.items;+ }+ return extract;+}+export async function extractData({extractOptions,urls,@@ -355,7 +372,16 @@ export async function extractData({}}- const { schemaToUse } = prepareSmartScrapeSchema(schema, logger, isSingleUrl);+ // The SmartScrape fields only matter when the agent can act on them.+ // Without it, the user's schema goes to generateCompletions as-is (with no+ // schema at all, the wrapper would require an extractedData property it+ // doesn't have). Nested under the wrapper, a bare property map is turned+ // into an object schema first, as generateCompletions does at the root.+ const wrapForSmartScrape = useAgent && !!schema;+ const schemaToUse = wrapForSmartScrape+ ? prepareSmartScrapeSchema(toRootSchema(schema), logger, isSingleUrl)+ .schemaToUse+ : schema;const extractOptionsNewSchema = {...extractOptions,options: { ...extractOptions.options, schema: schemaToUse },@@ -424,7 +450,9 @@ export async function extractData({warning = `JSON extraction failed: ${reason.slice(0, 300)}`;}- let extractedData = extract?.extractedData;+ let extractedData = wrapForSmartScrape+ ? extract?.extractedData+ : unwrapRootArray(schema, extract);// console.log("shouldUseSmartscrape", extract?.shouldUseSmartscrape);// console.log("smartscrape_reasoning", extract?.smartscrape_reasoning);@@ -438,7 +466,7 @@ export async function extractData({providedExtractId: extractId,});- if (useAgent && extract?.shouldUseSmartscrape) {+ if (wrapForSmartScrape && extract?.shouldUseSmartscrape) {// technically this should be checked upstream but might as well add another guard - Mogeryif (extractOptions.zeroDataRetention) {throw new Error(diff --git a/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.tsnew file mode 100644index 000000000..00ff91858--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/generateCompletions.test.ts@@ -0,0 +1,165 @@+import { vi } from "vitest";++vi.mock("ai", async importOriginal => {+ const actual = await importOriginal<typeof import("ai")>();+ return {+ ...actual,+ generateObject: vi.fn(async () => ({+ object: { ok: true },+ usage: { inputTokens: 1, outputTokens: 1 },+ })),+ };+});++import { generateObject } from "ai";+import { encoding_for_model } from "@dqbd/tiktoken";+import type { Mock } from "vitest";+import { generateCompletions } from "./llmExtract";+import { CostTracking } from "../../../lib/cost-tracking";+import { modelPrices } from "../../../lib/extract/usage/model-prices";++const noopLogger = {+ warn: () => {},+ info: () => {},+ error: () => {},+ debug: () => {},+ child: () => noopLogger,+} as any;++function run(markdown: string, modelId: string, schema?: any) {+ return generateCompletions({+ logger: noopLogger,+ options: { schema },+ markdown,+ model: { modelId } as any,+ retryModel: { modelId } as any,+ costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },+ metadata: { teamId: "test-team" },+ });+}++function lastCall() {+ const calls = (generateObject as Mock).mock.calls;+ expect(calls).toHaveLength(1);+ return calls[0][0];+}++function countTokens(text: string): number {+ const encoder = encoding_for_model("gpt-4o-mini");+ try {+ return encoder.encode(text).length;+ } finally {+ encoder.free();+ }+}++beforeEach(() => {+ (generateObject as Mock).mockClear();+});++describe("generateCompletions content trimming", () => {+ // ~3.3 characters per token like real pages: ~400k tokens, three times+ // gpt-4o-mini's window.+ const hugeMarkdown =+ "The quick brown fox jumps over the lazy dog. [link](https://example.com/a/b?c=1) 12345 ".repeat(+ 15_000,+ );++ it("trims content that would overflow the model's context window", async () => {+ const result = await run(hugeMarkdown, "gpt-4o-mini");++ expect(result.warning).toContain(+ "the input has been automatically trimmed",+ );+ // The content is trimmed to exactly 80% of the window; the rest of the+ // prompt is a short instruction.+ const budget = Math.floor(+ modelPrices["gpt-4o-mini"].max_input_tokens * 0.8,+ );+ const promptTokens = countTokens(lastCall().prompt);+ expect(promptTokens).toBeGreaterThan(budget);+ expect(promptTokens).toBeLessThanOrEqual(budget + 100);+ });++ it("leaves content that fits untouched", async () => {+ const markdown = "a small page";+ const result = await run(markdown, "gpt-4o-mini");++ expect(result.warning).toBeUndefined();+ expect(lastCall().prompt).toContain(markdown);+ });++ it("does not trim for models without known limits", async () => {+ const result = await run(hugeMarkdown, "some-unlisted-model");++ expect(result.warning).toBeUndefined();+ expect(lastCall().prompt).toContain(hugeMarkdown);+ });+});++describe("generateCompletions schema normalization", () => {+ it("turns a bare property map into an object schema, skipping $schema", async () => {+ await run("page", "gpt-4o-mini", {+ $schema: "http://json-schema.org/draft-07/schema#",+ judgments: {+ type: "array",+ items: {+ type: "object",+ properties: { citation: { type: "String" } },+ },+ },+ });++ expect(lastCall().schema.jsonSchema).toEqual({+ type: "object",+ properties: {+ judgments: {+ type: "array",+ items: {+ type: "object",+ properties: { citation: { type: "string" } },+ required: ["citation"],+ additionalProperties: false,+ },+ },+ },+ required: ["judgments"],+ additionalProperties: false,+ });+ });++ it("wraps nullable root arrays in an object", async () => {+ await run("page", "gpt-4o-mini", {+ type: ["array", "null"],+ items: { type: "string" },+ });++ expect(lastCall().schema.jsonSchema).toEqual({+ type: "object",+ properties: {+ items: { type: ["array", "null"], items: { type: "string" } },+ },+ required: ["items"],+ additionalProperties: false,+ });+ });++ it("closes nullable objects so strict mode accepts them", async () => {+ await run("page", "gpt-4o-mini", {+ type: "object",+ properties: {+ address: {+ type: ["object", "null"],+ properties: { city: { type: "string" } },+ },+ },+ });++ expect(lastCall().schema.jsonSchema.properties.address).toEqual({+ type: ["object", "null"],+ properties: { city: { type: "string" } },+ required: ["city"],+ additionalProperties: false,+ });+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex ee63de2be..08aa15d22 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -22,6 +22,11 @@ import { z } from "zod";import fs from "fs/promises";import Ajv from "ajv";import { extractData } from "../lib/extractSmartScrape";+import {+ findStrictSchemaViolation,+ toRootSchema,+ typeIncludes,+} from "../../../lib/openai-strict-schema";import { CostTracking } from "../../../lib/cost-tracking";import { isAgentExtractModelValid } from "../../../controllers/v1/types";import { hasFormatOfType } from "../../../lib/format-utils";@@ -131,7 +136,7 @@ function normalizeSchema(x: any): any {x.not = normalizeSchema(x.not);}- if (x && x.type === "object") {+ if (x && typeIncludes(x.type, "object")) {return {...x,properties: Object.fromEntries(@@ -143,7 +148,7 @@ function normalizeSchema(x: any): any {required: Object.keys(x.properties || {}),additionalProperties: false,};- } else if (x && x.type === "array") {+ } else if (x && typeIncludes(x.type, "array")) {return {...x,items: normalizeSchema(x.items),@@ -342,6 +347,20 @@ export async function generateCompletions({throw new Error("document.markdown is undefined -- this is unexpected");}+ // Keep the content inside the model's context window, leaving the rest for+ // the prompt, schema and output. Models without known limits are sent the+ // content as-is. A BPE token is at least one byte, so content that fits in+ // bytes skips the (synchronous) tokenizer entirely.+ const maxInputTokens = modelPrices[modelId]?.max_input_tokens;+ if (markdown && maxInputTokens) {+ const maxContentTokens = Math.floor(maxInputTokens * 0.8);+ if (Buffer.byteLength(markdown, "utf8") > maxContentTokens) {+ const trimmed = trimToTokenLimit(markdown, maxContentTokens, modelId);+ markdown = trimmed.text;+ warning = trimmed.warning;+ }+ }+try {const prompt =options.prompt !== undefined@@ -565,29 +584,21 @@ export async function generateCompletions({if (schema && !(schema instanceof z.ZodType)) {// let schema = options.schema;if (schema) {- schema = removeDefaultProperty(schema);+ schema = toRootSchema(removeDefaultProperty(schema));}- if (schema && schema.type === "array") {+ // Structured outputs need a (non-nullable) object at the root.+ if (schema && typeIncludes(schema.type, "array")) {schema = {type: "object",properties: {- items: options.schema,+ items: schema,},required: ["items"],additionalProperties: false,};- } else if (schema && typeof schema === "object" && !schema.type) {- schema = {- type: "object",- properties: Object.fromEntries(- Object.entries(schema).map(([key, value]) => {- return [key, removeDefaultProperty(value)];- }),- ),- required: Object.keys(schema),- additionalProperties: false,- };+ } else if (schema && typeIncludes(schema.type, "object")) {+ schema = { ...schema, type: "object" };}schema = normalizeSchema(schema);@@ -1469,6 +1480,9 @@ export async function generateSchemaFromPrompt(const retryModel = getModel("gpt-4.1-mini", "openai");const temperatures = [0, 0.1, 0.3]; // Different temperatures to trylet lastError: Error | null = null;+ // The last schema strict mode would reject, used only if no attempt+ // produces a supported one (extraction then fails with a warning as before).+ let lastUnsupportedSchema: any = undefined;for (const temp of temperatures) {try {@@ -1489,16 +1503,18 @@ Consider:Valid JSON schema, has to be simple. No crazy properties. OpenAI has to support it.Supported types-The following types are supported for Structured Outputs:+The following values of "type" are supported for Structured Outputs, written exactly like this (lowercase):++string+number+boolean+integer+object+array-String-Number-Boolean-Integer-Object-Array-Enum-anyOf+Use "enum" as a keyword next to "type": "string" to restrict values; it is not a type. Use "anyOf" for alternatives.+Every array must define "items" as a single schema object. Every property must be a schema object with a "type".+allOf, oneOf, not, and if/then/else are not supported.Formats are not supported. Min/max are not supported. Anything beyond the above is not supported. Keep it simple with types and descriptions.Optionals are not supported.@@ -1524,7 +1540,15 @@ Return a valid JSON schema object with properties that would capture the informazeroDataRetention,});- return { extract };+ const violation = findStrictSchemaViolation(extract);+ if (violation === null) {+ return { extract };+ }+ logger.warn("Generated schema is not supported by structured outputs", {+ violation,+ temperature: temp,+ });+ lastUnsupportedSchema = extract;} catch (error) {lastError = error as Error;logger.warn(`Failed attempt with temperature ${temp}: ${error.message}`);@@ -1532,6 +1556,10 @@ Return a valid JSON schema object with properties that would capture the informa}}+ if (lastUnsupportedSchema !== undefined) {+ return { extract: lastUnsupportedSchema };+ }+// If we get here, all attempts failedthrow new Error(`Failed to generate schema after all attempts. Last error: ${lastError?.message}`,