Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: handle `$ref` for recursive schema validation (#2238)

Gaurav Chadha10e07cb04dcd
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index dd7c0c168..8f4a75762 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -72,13 +72,39 @@ function normalizeSchemaForOpenAI(schema: any): any {
function normalizeObject(obj: any): any {
if (typeof obj !== "object" || obj === null) return obj;
- if (Array.isArray(obj)) return obj;
+ if (Array.isArray(obj)) {
+ return obj.map(item => normalizeObject(item));
+ }
if (visited.has(obj)) return obj;
visited.add(obj);
const normalized = { ...obj };
+ // Handle $ref recursion - preserve as-is for OpenAI compatibility
+ if (normalized.hasOwnProperty("$ref")) {
+ return normalized;
+ }
+
+ if (normalized.hasOwnProperty("$defs")) {
+ const { $defs, ...rest } = normalized;
+ const processedRest = {};
+
+ for (const [key, value] of Object.entries(rest)) {
+ if (
+ typeof value === "object" &&
+ value !== null &&
+ !value.hasOwnProperty("$ref")
+ ) {
+ processedRest[key] = normalizeObject(value);
+ } else {
+ processedRest[key] = value;
+ }
+ }
+
+ return { ...processedRest, $defs };
+ }
+
if (
normalized.type === "object" &&
normalized.hasOwnProperty("properties") &&
@@ -111,7 +137,11 @@ function normalizeSchemaForOpenAI(schema: any): any {
}
for (const [key, value] of Object.entries(normalized)) {
- if (typeof value === "object" && value !== null) {
+ if (
+ typeof value === "object" &&
+ value !== null &&
+ !value.hasOwnProperty("$ref")
+ ) {
normalized[key] = normalizeObject(value);
}
}
@@ -135,6 +165,10 @@ function validateSchemaForOpenAI(schema: any): boolean {
if (visited.has(obj)) return false;
visited.add(obj);
+ if (obj.hasOwnProperty("$ref")) {
+ return false;
+ }
+
if (
obj.type === "object" &&
!obj.hasOwnProperty("properties") &&
@@ -145,7 +179,11 @@ function validateSchemaForOpenAI(schema: any): boolean {
}
for (const value of Object.values(obj)) {
- if (typeof value === "object" && value !== null) {
+ if (
+ typeof value === "object" &&
+ value !== null &&
+ !value.hasOwnProperty("$ref")
+ ) {
if (hasInvalidStructure(value)) return true;
}
}
@@ -156,7 +194,7 @@ function validateSchemaForOpenAI(schema: any): boolean {
}
const OPENAI_SCHEMA_ERROR_MESSAGE =
- "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";
+ "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object. Note: Recursive schemas using '$ref' are supported.";
const ACTIONS_MAX_WAIT_TIME = 60;
const MAX_ACTIONS = 50;
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 3088a46c2..d4b23c8bd 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -120,29 +120,114 @@ function prepareSmartScrapeSchema(
return { schemaToUse: wrappedSchema };
}
+const hasRecursiveRefs = (schema: any, defs: any): boolean => {
+ if (!defs || typeof defs !== "object") return false;
+
+ for (const [defName, defValue] of Object.entries(defs)) {
+ if (containsRecursiveRef(defValue, defName, defs)) {
+ return true;
+ }
+ }
+
+ return false;
+};
+
+const containsRecursiveRef = (
+ obj: any,
+ targetDefName: string,
+ defs: any,
+ visited = new Set(),
+): boolean => {
+ if (!obj || typeof obj !== "object") return false;
+
+ const objKey = JSON.stringify(obj);
+ if (visited.has(objKey)) return false;
+ visited.add(objKey);
+
+ if (obj.$ref && typeof obj.$ref === "string") {
+ const refPath = obj.$ref.split("/");
+ if (refPath[0] === "#" && refPath[1] === "$defs") {
+ const defName = refPath[refPath.length - 1];
+ if (defName === targetDefName) {
+ visited.delete(objKey);
+ return true;
+ }
+ if (defs[defName]) {
+ const isRecursive = containsRecursiveRef(
+ defs[defName],
+ targetDefName,
+ defs,
+ visited,
+ );
+ visited.delete(objKey);
+ return isRecursive;
+ }
+ }
+ }
+
+ if (Array.isArray(obj)) {
+ for (const item of obj) {
+ if (containsRecursiveRef(item, targetDefName, defs, visited)) {
+ visited.delete(objKey);
+ return true;
+ }
+ }
+ } else {
+ for (const value of Object.values(obj)) {
+ if (containsRecursiveRef(value, targetDefName, defs, visited)) {
+ visited.delete(objKey);
+ return true;
+ }
+ }
+ }
+
+ visited.delete(objKey);
+ return false;
+};
+
// Resolve all $defs references in the schema
-const resolveRefs = (obj: any, defs: any): any => {
- if (!obj || typeof obj !== "object") return obj;
+const resolveRefs = (
+ obj: any,
+ defs: any,
+ visited = new WeakSet(),
+ depth = 0,
+): any => {
+ if (!obj || typeof obj !== "object" || depth > 10) return obj;
+
+ const objString = JSON.stringify(obj);
+ if (objString.includes("#/$defs/") && objString.includes('"$ref"')) {
+ console.warn(
+ "resolveRefs: Detected recursive schema pattern, aborting to prevent infinite recursion",
+ );
+ return obj;
+ }
+
+ // Prevent infinite recursion
+ if (visited.has(obj)) return obj;
+ visited.add(obj);
if (obj.$ref && typeof obj.$ref === "string") {
// Handle $ref references
const refPath = obj.$ref.split("/");
if (refPath[0] === "#" && refPath[1] === "$defs") {
const defName = refPath[refPath.length - 1];
- return resolveRefs({ ...defs[defName] }, defs);
+ if (defs[defName]) {
+ return resolveRefs({ ...defs[defName] }, defs, visited, depth + 1);
+ }
}
+ return obj; // Return original if ref can't be resolved
}
// Handle arrays
if (Array.isArray(obj)) {
- return obj.map(item => resolveRefs(item, defs));
+ return obj.map(item => resolveRefs(item, defs, visited, depth + 1));
}
// Handle objects
const resolved: any = {};
for (const [key, value] of Object.entries(obj)) {
if (key === "$defs") continue;
- resolved[key] = resolveRefs(value, defs);
+ resolved[key] = resolveRefs(value, defs, visited, depth + 1);
}
return resolved;
};
@@ -193,11 +278,31 @@ export async function extractData({
if (schema) {
const defs = schema.$defs || {};
- schema = resolveRefs(schema, defs);
- delete schema.$defs;
- logger.info("Resolved schema refs", {
- schema,
- });
+ const schemaString = JSON.stringify(schema);
+ const hasAnyRefs =
+ schema.$defs ||
+ schemaString.includes('"$ref"') ||
+ schemaString.includes("#/$defs/");
+
+ if (hasAnyRefs) {
+ logger.info(
+ "Detected schema with references, preserving as-is to avoid recursion",
+ {
+ hasDefsProperty: !!schema.$defs,
+ hasRefInString: schemaString.includes('"$ref"'),
+ hasRefPathInString: schemaString.includes("#/$defs/"),
+ },
+ );
+ } else {
+ logger.info("No recursive references detected, resolving refs", {
+ schema,
+ });
+ schema = resolveRefs(schema, defs);
+ delete schema.$defs;
+ logger.info("Resolved schema refs", {
+ schema,
+ });
+ }
}
const { schemaToUse } = prepareSmartScrapeSchema(schema, logger, isSingleUrl);
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index e57963882..27cb616ef 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -25,6 +25,46 @@ import { extractData } from "../lib/extractSmartScrape";
import { CostTracking } from "../../../lib/cost-tracking";
import { isAgentExtractModelValid } from "../../../controllers/v1/types";
import { hasFormatOfType } from "../../../lib/format-utils";
+
+// Smart model selection based on schema
+function detectRecursiveSchema(schema: any): boolean {
+ if (!schema || typeof schema !== "object") return false;
+
+ const schemaString = JSON.stringify(schema);
+ const hasRefs =
+ schemaString.includes('"$ref"') ||
+ schemaString.includes("#/$defs/") ||
+ schemaString.includes("#/definitions/");
+ const hasDefs = !!(schema.$defs || schema.definitions);
+
+ return hasRefs || hasDefs;
+}
+
+function selectModelForSchema(schema?: any): {
+ modelName: string;
+ reason: string;
+} {
+ if (!schema) {
+ return { modelName: "gpt-4o-mini", reason: "no_schema" };
+ }
+
+ const isRecursive = detectRecursiveSchema(schema);
+
+ if (isRecursive) {
+ logger.info(`Model: gpt-4o | hasRef: true`);
+ return {
+ modelName: "gpt-4o",
+ reason: "recursive_schema_detected",
+ };
+ }
+
+ logger.info(`Model: gpt-4o-mini | hasRef: false`);
+ return {
+ modelName: "gpt-4o-mini",
+ reason: "simple_schema",
+ };
+}
+
// TODO: fix this, it's horrible
type LanguageModelV1ProviderMetadata = {
anthropic?: {
@@ -890,6 +930,8 @@ export async function performLLMExtract(
// let generationOptions = { ...originalOptions }; // Start with original options
+ const modelSelection = selectModelForSchema(jsonFormat.schema);
+
const generationOptions: GenerateCompletionsOptions = {
logger: meta.logger.child({
method: "performLLMExtract/generateCompletions",
@@ -897,7 +939,7 @@ export async function performLLMExtract(
options: jsonFormat,
markdown: document.markdown,
previousWarning: document.warning,
- model: getModel("gpt-4o-mini", "openai"),
+ model: getModel(modelSelection.modelName, "openai"),
retryModel: getModel("gpt-4o", "openai"),
costTrackingOptions: {
costTracking: meta.costTracking,
@@ -1089,7 +1131,15 @@ export async function performSummary(
},
markdown: trimOutput.text,
previousWarning: document.warning,
- model: getModel("gpt-4o-mini", "openai"),
+ model: (() => {
+ const inlineSchema = {
+ type: "object",
+ properties: { summary: { type: "string" } },
+ required: ["summary"],
+ };
+ const selection = selectModelForSchema(inlineSchema);
+ return getModel(selection.modelName, "openai");
+ })(),
retryModel: getModel("gpt-4o", "openai"),
costTrackingOptions: {
costTracking: meta.costTracking,