Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
update: handle circular refs as well in recursive schema (#2298)
Gaurav Chadhaa35e7fad0a4a
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 4cd9d707c..592c4dc13 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -110,7 +110,14 @@ function normalizeSchemaForOpenAI(schema: any): any {}}- return { ...processedRest, $defs };+ const normalizedDefs = Object.fromEntries(+ Object.entries($defs ?? {}).map(([key, value]) => [+ key,+ normalizeObject(value)+ ])+ );++ return { ...processedRest, $defs: normalizedDefs };}if (diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex d4b23c8bd..549a58221 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -189,21 +189,18 @@ const containsRecursiveRef = (const resolveRefs = (obj: any,defs: any,+ logger: Logger,visited = new WeakSet(),depth = 0,): any => {if (!obj || typeof obj !== "object" || depth > 10) return obj;- const objString = JSON.stringify(obj);- if (objString.includes("#/$defs/") && objString.includes('"$ref"')) {- console.warn(- "resolveRefs: Detected recursive schema pattern, aborting to prevent infinite recursion",+ if (visited.has(obj)) {+ logger.warn(+ "resolveRefs: Detected circular reference, aborting to prevent infinite recursion",);return obj;}-- // Prevent infinite recursion- if (visited.has(obj)) return obj;visited.add(obj);if (obj.$ref && typeof obj.$ref === "string") {@@ -212,7 +209,7 @@ const resolveRefs = (if (refPath[0] === "#" && refPath[1] === "$defs") {const defName = refPath[refPath.length - 1];if (defs[defName]) {- return resolveRefs({ ...defs[defName] }, defs, visited, depth + 1);+ return resolveRefs({ ...defs[defName] }, defs, logger, visited, depth + 1);}}return obj; // Return original if ref can't be resolved@@ -220,14 +217,14 @@ const resolveRefs = (// Handle arraysif (Array.isArray(obj)) {- return obj.map(item => resolveRefs(item, defs, visited, depth + 1));+ return obj.map(item => resolveRefs(item, defs, logger, visited, depth + 1));}// Handle objectsconst resolved: any = {};for (const [key, value] of Object.entries(obj)) {if (key === "$defs") continue;- resolved[key] = resolveRefs(value, defs, visited, depth + 1);+ resolved[key] = resolveRefs(value, defs, logger, visited, depth + 1);}return resolved;};@@ -286,18 +283,37 @@ export async function extractData({if (hasAnyRefs) {logger.info(- "Detected schema with references, preserving as-is to avoid recursion",+ "Detected schema with references, attempting to resolve refs",{hasDefsProperty: !!schema.$defs,hasRefInString: schemaString.includes('"$ref"'),hasRefPathInString: schemaString.includes("#/$defs/"),},);++ try {+ const resolvedSchema = resolveRefs(schema, defs, logger);++ const resolvedString = JSON.stringify(resolvedSchema);+ const hasRemainingRefs = resolvedString.includes('"$ref"') || resolvedString.includes("#/$defs/");++ if (!hasRemainingRefs) {+ schema = resolvedSchema;+ if (schema && typeof schema === "object" && schema.$defs) delete schema.$defs;+ logger.info("Successfully resolved schema refs", {+ schema,+ });+ } else {+ logger.info("Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema");+ }+ } catch (error) {+ logger.warn("Failed to resolve schema refs, preserving original schema", { error });+ }} else {logger.info("No recursive references detected, resolving refs", {schema,});- schema = resolveRefs(schema, defs);+ schema = resolveRefs(schema, defs, logger);delete schema.$defs;logger.info("Resolved schema refs", {schema,