Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: handle `$ref` for recursive schema validation (#2238)
Gaurav Chadha10e07cb04dcd
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex dd7c0c168..8f4a75762 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -72,13 +72,39 @@ function normalizeSchemaForOpenAI(schema: any): any {function normalizeObject(obj: any): any {if (typeof obj !== "object" || obj === null) return obj;- if (Array.isArray(obj)) return obj;+ if (Array.isArray(obj)) {+ return obj.map(item => normalizeObject(item));+ }if (visited.has(obj)) return obj;visited.add(obj);const normalized = { ...obj };+ // Handle $ref recursion - preserve as-is for OpenAI compatibility+ if (normalized.hasOwnProperty("$ref")) {+ return normalized;+ }++ if (normalized.hasOwnProperty("$defs")) {+ const { $defs, ...rest } = normalized;+ const processedRest = {};++ for (const [key, value] of Object.entries(rest)) {+ if (+ typeof value === "object" &&+ value !== null &&+ !value.hasOwnProperty("$ref")+ ) {+ processedRest[key] = normalizeObject(value);+ } else {+ processedRest[key] = value;+ }+ }++ return { ...processedRest, $defs };+ }+if (normalized.type === "object" &&normalized.hasOwnProperty("properties") &&@@ -111,7 +137,11 @@ function normalizeSchemaForOpenAI(schema: any): any {}for (const [key, value] of Object.entries(normalized)) {- if (typeof value === "object" && value !== null) {+ if (+ typeof value === "object" &&+ value !== null &&+ !value.hasOwnProperty("$ref")+ ) {normalized[key] = normalizeObject(value);}}@@ -135,6 +165,10 @@ function validateSchemaForOpenAI(schema: any): boolean {if (visited.has(obj)) return false;visited.add(obj);+ if (obj.hasOwnProperty("$ref")) {+ return false;+ }+if (obj.type === "object" &&!obj.hasOwnProperty("properties") &&@@ -145,7 +179,11 @@ function validateSchemaForOpenAI(schema: any): boolean {}for (const value of Object.values(obj)) {- if (typeof value === "object" && value !== null) {+ if (+ typeof value === "object" &&+ value !== null &&+ !value.hasOwnProperty("$ref")+ ) {if (hasInvalidStructure(value)) return true;}}@@ -156,7 +194,7 @@ function validateSchemaForOpenAI(schema: any): boolean {}const OPENAI_SCHEMA_ERROR_MESSAGE =- "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";+ "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object. Note: Recursive schemas using '$ref' are supported.";const ACTIONS_MAX_WAIT_TIME = 60;const MAX_ACTIONS = 50;diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex 3088a46c2..d4b23c8bd 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -120,29 +120,114 @@ function prepareSmartScrapeSchema(return { schemaToUse: wrappedSchema };}+const hasRecursiveRefs = (schema: any, defs: any): boolean => {+ if (!defs || typeof defs !== "object") return false;++ for (const [defName, defValue] of Object.entries(defs)) {+ if (containsRecursiveRef(defValue, defName, defs)) {+ return true;+ }+ }++ return false;+};++const containsRecursiveRef = (+ obj: any,+ targetDefName: string,+ defs: any,+ visited = new Set(),+): boolean => {+ if (!obj || typeof obj !== "object") return false;++ const objKey = JSON.stringify(obj);+ if (visited.has(objKey)) return false;+ visited.add(objKey);++ if (obj.$ref && typeof obj.$ref === "string") {+ const refPath = obj.$ref.split("/");+ if (refPath[0] === "#" && refPath[1] === "$defs") {+ const defName = refPath[refPath.length - 1];+ if (defName === targetDefName) {+ visited.delete(objKey);+ return true;+ }+ if (defs[defName]) {+ const isRecursive = containsRecursiveRef(+ defs[defName],+ targetDefName,+ defs,+ visited,+ );+ visited.delete(objKey);+ return isRecursive;+ }+ }+ }++ if (Array.isArray(obj)) {+ for (const item of obj) {+ if (containsRecursiveRef(item, targetDefName, defs, visited)) {+ visited.delete(objKey);+ return true;+ }+ }+ } else {+ for (const value of Object.values(obj)) {+ if (containsRecursiveRef(value, targetDefName, defs, visited)) {+ visited.delete(objKey);+ return true;+ }+ }+ }++ visited.delete(objKey);+ return false;+};+// Resolve all $defs references in the schema-const resolveRefs = (obj: any, defs: any): any => {- if (!obj || typeof obj !== "object") return obj;+const resolveRefs = (+ obj: any,+ defs: any,+ visited = new WeakSet(),+ depth = 0,+): any => {+ if (!obj || typeof obj !== "object" || depth > 10) return obj;++ const objString = JSON.stringify(obj);+ if (objString.includes("#/$defs/") && objString.includes('"$ref"')) {+ console.warn(+ "resolveRefs: Detected recursive schema pattern, aborting to prevent infinite recursion",+ );+ return obj;+ }++ // Prevent infinite recursion+ if (visited.has(obj)) return obj;+ visited.add(obj);if (obj.$ref && typeof obj.$ref === "string") {// Handle $ref referencesconst refPath = obj.$ref.split("/");if (refPath[0] === "#" && refPath[1] === "$defs") {const defName = refPath[refPath.length - 1];- return resolveRefs({ ...defs[defName] }, defs);+ if (defs[defName]) {+ return resolveRefs({ ...defs[defName] }, defs, visited, depth + 1);+ }}+ return obj; // Return original if ref can't be resolved}// Handle arraysif (Array.isArray(obj)) {- return obj.map(item => resolveRefs(item, defs));+ return obj.map(item => resolveRefs(item, defs, visited, depth + 1));}// Handle objectsconst resolved: any = {};for (const [key, value] of Object.entries(obj)) {if (key === "$defs") continue;- resolved[key] = resolveRefs(value, defs);+ resolved[key] = resolveRefs(value, defs, visited, depth + 1);}return resolved;};@@ -193,11 +278,31 @@ export async function extractData({if (schema) {const defs = schema.$defs || {};- schema = resolveRefs(schema, defs);- delete schema.$defs;- logger.info("Resolved schema refs", {- schema,- });+ const schemaString = JSON.stringify(schema);+ const hasAnyRefs =+ schema.$defs ||+ schemaString.includes('"$ref"') ||+ schemaString.includes("#/$defs/");++ if (hasAnyRefs) {+ logger.info(+ "Detected schema with references, preserving as-is to avoid recursion",+ {+ hasDefsProperty: !!schema.$defs,+ hasRefInString: schemaString.includes('"$ref"'),+ hasRefPathInString: schemaString.includes("#/$defs/"),+ },+ );+ } else {+ logger.info("No recursive references detected, resolving refs", {+ schema,+ });+ schema = resolveRefs(schema, defs);+ delete schema.$defs;+ logger.info("Resolved schema refs", {+ schema,+ });+ }}const { schemaToUse } = prepareSmartScrapeSchema(schema, logger, isSingleUrl);diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex e57963882..27cb616ef 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -25,6 +25,46 @@ import { extractData } from "../lib/extractSmartScrape";import { CostTracking } from "../../../lib/cost-tracking";import { isAgentExtractModelValid } from "../../../controllers/v1/types";import { hasFormatOfType } from "../../../lib/format-utils";++// Smart model selection based on schema+function detectRecursiveSchema(schema: any): boolean {+ if (!schema || typeof schema !== "object") return false;++ const schemaString = JSON.stringify(schema);+ const hasRefs =+ schemaString.includes('"$ref"') ||+ schemaString.includes("#/$defs/") ||+ schemaString.includes("#/definitions/");+ const hasDefs = !!(schema.$defs || schema.definitions);++ return hasRefs || hasDefs;+}++function selectModelForSchema(schema?: any): {+ modelName: string;+ reason: string;+} {+ if (!schema) {+ return { modelName: "gpt-4o-mini", reason: "no_schema" };+ }++ const isRecursive = detectRecursiveSchema(schema);++ if (isRecursive) {+ logger.info(`Model: gpt-4o | hasRef: true`);+ return {+ modelName: "gpt-4o",+ reason: "recursive_schema_detected",+ };+ }++ logger.info(`Model: gpt-4o-mini | hasRef: false`);+ return {+ modelName: "gpt-4o-mini",+ reason: "simple_schema",+ };+}+// TODO: fix this, it's horribletype LanguageModelV1ProviderMetadata = {anthropic?: {@@ -890,6 +930,8 @@ export async function performLLMExtract(// let generationOptions = { ...originalOptions }; // Start with original options+ const modelSelection = selectModelForSchema(jsonFormat.schema);+const generationOptions: GenerateCompletionsOptions = {logger: meta.logger.child({method: "performLLMExtract/generateCompletions",@@ -897,7 +939,7 @@ export async function performLLMExtract(options: jsonFormat,markdown: document.markdown,previousWarning: document.warning,- model: getModel("gpt-4o-mini", "openai"),+ model: getModel(modelSelection.modelName, "openai"),retryModel: getModel("gpt-4o", "openai"),costTrackingOptions: {costTracking: meta.costTracking,@@ -1089,7 +1131,15 @@ export async function performSummary(},markdown: trimOutput.text,previousWarning: document.warning,- model: getModel("gpt-4o-mini", "openai"),+ model: (() => {+ const inlineSchema = {+ type: "object",+ properties: { summary: { type: "string" } },+ required: ["summary"],+ };+ const selection = selectModelForSchema(inlineSchema);+ return getModel(selection.modelName, "openai");+ })(),retryModel: getModel("gpt-4o", "openai"),costTrackingOptions: {costTracking: meta.costTracking,