Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

update: handle circular refs as well in recursive schema (#2298)

Gaurav Chadhaa35e7fad0a4a
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 4cd9d707c..592c4dc13 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -110,7 +110,14 @@ function normalizeSchemaForOpenAI(schema: any): any {
}
}
- return { ...processedRest, $defs };
+ const normalizedDefs = Object.fromEntries(
+ Object.entries($defs ?? {}).map(([key, value]) => [
+ key,
+ normalizeObject(value)
+ ])
+ );
+
+ return { ...processedRest, $defs: normalizedDefs };
}
if (
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index d4b23c8bd..549a58221 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -189,21 +189,18 @@ const containsRecursiveRef = (
const resolveRefs = (
obj: any,
defs: any,
+ logger: Logger,
visited = new WeakSet(),
depth = 0,
): any => {
if (!obj || typeof obj !== "object" || depth > 10) return obj;
- const objString = JSON.stringify(obj);
- if (objString.includes("#/$defs/") && objString.includes('"$ref"')) {
- console.warn(
- "resolveRefs: Detected recursive schema pattern, aborting to prevent infinite recursion",
+ if (visited.has(obj)) {
+ logger.warn(
+ "resolveRefs: Detected circular reference, aborting to prevent infinite recursion",
);
return obj;
}
-
- // Prevent infinite recursion
- if (visited.has(obj)) return obj;
visited.add(obj);
if (obj.$ref && typeof obj.$ref === "string") {
@@ -212,7 +209,7 @@ const resolveRefs = (
if (refPath[0] === "#" && refPath[1] === "$defs") {
const defName = refPath[refPath.length - 1];
if (defs[defName]) {
- return resolveRefs({ ...defs[defName] }, defs, visited, depth + 1);
+ return resolveRefs({ ...defs[defName] }, defs, logger, visited, depth + 1);
}
}
return obj; // Return original if ref can't be resolved
@@ -220,14 +217,14 @@ const resolveRefs = (
// Handle arrays
if (Array.isArray(obj)) {
- return obj.map(item => resolveRefs(item, defs, visited, depth + 1));
+ return obj.map(item => resolveRefs(item, defs, logger, visited, depth + 1));
}
// Handle objects
const resolved: any = {};
for (const [key, value] of Object.entries(obj)) {
if (key === "$defs") continue;
- resolved[key] = resolveRefs(value, defs, visited, depth + 1);
+ resolved[key] = resolveRefs(value, defs, logger, visited, depth + 1);
}
return resolved;
};
@@ -286,18 +283,37 @@ export async function extractData({
if (hasAnyRefs) {
logger.info(
- "Detected schema with references, preserving as-is to avoid recursion",
+ "Detected schema with references, attempting to resolve refs",
{
hasDefsProperty: !!schema.$defs,
hasRefInString: schemaString.includes('"$ref"'),
hasRefPathInString: schemaString.includes("#/$defs/"),
},
);
+
+ try {
+ const resolvedSchema = resolveRefs(schema, defs, logger);
+
+ const resolvedString = JSON.stringify(resolvedSchema);
+ const hasRemainingRefs = resolvedString.includes('"$ref"') || resolvedString.includes("#/$defs/");
+
+ if (!hasRemainingRefs) {
+ schema = resolvedSchema;
+ if (schema && typeof schema === "object" && schema.$defs) delete schema.$defs;
+ logger.info("Successfully resolved schema refs", {
+ schema,
+ });
+ } else {
+ logger.info("Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema");
+ }
+ } catch (error) {
+ logger.warn("Failed to resolve schema refs, preserving original schema", { error });
+ }
} else {
logger.info("No recursive references detected, resolving refs", {
schema,
});
- schema = resolveRefs(schema, defs);
+ schema = resolveRefs(schema, defs, logger);
delete schema.$defs;
logger.info("Resolved schema refs", {
schema,