Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(extract): replace json-schema-ref-parser with internal-only $ref resolver
tomsideguide053630fc5203
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 0ca1c9174..2a9337a35 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -81,7 +81,6 @@"@ai-sdk/groq": "^3.0.41","@ai-sdk/openai": "3.0.71","@ai-sdk/xai": "^3.0.83",- "@apidevtools/json-schema-ref-parser": "^15.1.2","@bull-board/api": "^6.14.0","@bull-board/express": "^6.14.0","@clickhouse/client": "^1.8.1",diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 1ac0ddefe..fecfa4b27 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -23,7 +23,7 @@ import {generateCompletions_F0,generateSchemaFromPrompt_F0,} from "./llmExtract-f0";-import { dereferenceSchema_F0 } from "./helpers/dereference-schema-f0";+import { dereferenceSchema } from "../helpers/dereference-schema";import { analyzeSchemaAndPrompt_F0 } from "./completions/analyzeSchemaAndPrompt-f0";import { checkShouldExtract_F0 } from "./completions/checkShouldExtract-f0";import { batchExtractPromise_F0 } from "./completions/batchExtract-f0";@@ -250,7 +250,7 @@ export async function performExtraction_F0(}if (reqSchema) {- reqSchema = await dereferenceSchema_F0(reqSchema);+ reqSchema = await dereferenceSchema(reqSchema);}logger.debug("Transformed schema.", {diff --git a/apps/api/src/lib/extract/fire-0/helpers/dereference-schema-f0.ts b/apps/api/src/lib/extract/fire-0/helpers/dereference-schema-f0.tsdeleted file mode 100644index d4f863a7b..000000000--- a/apps/api/src/lib/extract/fire-0/helpers/dereference-schema-f0.ts+++ /dev/null@@ -1,10 +0,0 @@-import { dereference } from "@apidevtools/json-schema-ref-parser";--export async function dereferenceSchema_F0(schema: any): Promise<any> {- try {- return await dereference(schema);- } catch (error) {- console.error("Failed to dereference schema:", error);- throw error;- }-}diff --git a/apps/api/src/lib/extract/helpers/__tests__/dereference-schema.test.ts b/apps/api/src/lib/extract/helpers/__tests__/dereference-schema.test.tsnew file mode 100644index 000000000..7730d56c2--- /dev/null+++ b/apps/api/src/lib/extract/helpers/__tests__/dereference-schema.test.ts@@ -0,0 +1,105 @@+import { dereferenceSchema } from "../dereference-schema";++describe("dereferenceSchema", () => {+ it("inlines an internal $ref pointer", async () => {+ const schema = {+ type: "object",+ properties: {+ user: { $ref: "#/definitions/User" },+ },+ definitions: {+ User: { type: "object", properties: { name: { type: "string" } } },+ },+ };++ const result = await dereferenceSchema(schema);++ expect(result.properties.user).toEqual({+ type: "object",+ properties: { name: { type: "string" } },+ });+ });++ it("resolves nested refs (a ref target that itself contains a ref)", async () => {+ const schema = {+ properties: { a: { $ref: "#/definitions/A" } },+ definitions: {+ A: { properties: { b: { $ref: "#/definitions/B" } } },+ B: { type: "number" },+ },+ };++ const result = await dereferenceSchema(schema);++ expect(result.properties.a.properties.b).toEqual({ type: "number" });+ });++ it("does not mutate the input schema", async () => {+ const schema = {+ properties: { user: { $ref: "#/definitions/User" } },+ definitions: { User: { type: "string" } },+ };++ await dereferenceSchema(schema);++ expect(schema.properties.user).toEqual({ $ref: "#/definitions/User" });+ });++ it("breaks self-referential cycles without infinite recursion", async () => {+ const schema = {+ definitions: {+ Node: {+ type: "object",+ properties: { next: { $ref: "#/definitions/Node" } },+ },+ },+ properties: { root: { $ref: "#/definitions/Node" } },+ };++ const result = await dereferenceSchema(schema);++ // One level is inlined; the cycle back to Node is left as a $ref.+ expect(result.properties.root.type).toBe("object");+ expect(result.properties.root.properties.next).toEqual({+ $ref: "#/definitions/Node",+ });+ });++ // Security: the whole reason this resolver exists. A $ref must NEVER be+ // resolved against the filesystem or network — only inside the document.+ it("leaves a file:// $ref untouched and reads nothing", async () => {+ const schema = {+ type: "object",+ properties: {+ x: { default: { $ref: "/etc/passwd#/nonexistent" } },+ },+ };++ const result = await dereferenceSchema(schema);++ // The external ref is passed through verbatim, not resolved to file contents.+ expect(result.properties.x.default).toEqual({+ $ref: "/etc/passwd#/nonexistent",+ });+ });++ it("leaves an http(s):// $ref untouched (no SSRF)", async () => {+ const schema = {+ properties: { y: { $ref: "http://169.254.169.254/latest/meta-data/" } },+ };++ const result = await dereferenceSchema(schema);++ expect(result.properties.y).toEqual({+ $ref: "http://169.254.169.254/latest/meta-data/",+ });+ });++ it("leaves an unresolvable internal pointer as-is", async () => {+ const schema = { properties: { z: { $ref: "#/definitions/Missing" } } };++ const result = await dereferenceSchema(schema);++ expect(result.properties.z).toEqual({ $ref: "#/definitions/Missing" });+ });+});diff --git a/apps/api/src/lib/extract/helpers/dereference-schema.ts b/apps/api/src/lib/extract/helpers/dereference-schema.tsindex 66f8fa504..895d83012 100644--- a/apps/api/src/lib/extract/helpers/dereference-schema.ts+++ b/apps/api/src/lib/extract/helpers/dereference-schema.ts@@ -1,8 +1,73 @@-import { dereference } from "@apidevtools/json-schema-ref-parser";+const MAX_REF_RESOLUTIONS = 50_000;++function unescapePointerSegment(segment: string): string {+ // JSON Pointer escaping: "~1" -> "/", "~0" -> "~" (order matters).+ return segment.replace(/~1/g, "/").replace(/~0/g, "~");+}++function resolveJsonPointer(root: any, ref: string): any {+ if (ref === "#") return root;+ // Only plain JSON-pointer fragments are supported. Anchor refs ("#foo") and+ // anything else are deliberately left unresolved rather than guessed at.+ if (!ref.startsWith("#/")) return undefined;++ let current = root;+ for (const segment of ref.slice(2).split("/").map(unescapePointerSegment)) {+ if (current === null || typeof current !== "object") return undefined;+ current = current[segment];+ if (current === undefined) return undefined;+ }+ return current;+}++function isRefObject(node: any): node is { $ref: string } {+ return (+ node !== null &&+ typeof node === "object" &&+ !Array.isArray(node) &&+ typeof node.$ref === "string"+ );+}export async function dereferenceSchema(schema: any): Promise<any> {+ const root = schema;+ let resolutions = 0;++ // `activeRefs` holds the pointers currently being expanded along this path;+ // revisiting one means a cycle, which we break by leaving the $ref in place.+ function walk(node: any, activeRefs: Set<string>): any {+ if (Array.isArray(node)) {+ return node.map(item => walk(item, activeRefs));+ }+ if (node === null || typeof node !== "object") {+ return node;+ }+ if (isRefObject(node)) {+ const ref = node.$ref;+ // External ref, cycle, or unresolvable pointer: leave the node as-is.+ if (!ref.startsWith("#") || activeRefs.has(ref)) {+ return { ...node };+ }+ const target = resolveJsonPointer(root, ref);+ if (target === undefined) {+ return { ...node };+ }+ if (++resolutions > MAX_REF_RESOLUTIONS) {+ throw new Error("Schema $ref resolution limit exceeded");+ }+ const nextActive = new Set(activeRefs);+ nextActive.add(ref);+ return walk(target, nextActive);+ }+ const result: Record<string, any> = {};+ for (const key of Object.keys(node)) {+ result[key] = walk(node[key], activeRefs);+ }+ return result;+ }+try {- return await dereference(schema);+ return walk(root, new Set());} catch (error) {console.error("Failed to dereference schema:", error);throw error;