Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(extract): replace json-schema-ref-parser with internal-only $ref resolver

tomsideguide053630fc5203
diff --git a/apps/api/package.json b/apps/api/package.json
index 0ca1c9174..2a9337a35 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -81,7 +81,6 @@
"@ai-sdk/groq": "^3.0.41",
"@ai-sdk/openai": "3.0.71",
"@ai-sdk/xai": "^3.0.83",
- "@apidevtools/json-schema-ref-parser": "^15.1.2",
"@bull-board/api": "^6.14.0",
"@bull-board/express": "^6.14.0",
"@clickhouse/client": "^1.8.1",
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index 1ac0ddefe..fecfa4b27 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -23,7 +23,7 @@ import {
generateCompletions_F0,
generateSchemaFromPrompt_F0,
} from "./llmExtract-f0";
-import { dereferenceSchema_F0 } from "./helpers/dereference-schema-f0";
+import { dereferenceSchema } from "../helpers/dereference-schema";
import { analyzeSchemaAndPrompt_F0 } from "./completions/analyzeSchemaAndPrompt-f0";
import { checkShouldExtract_F0 } from "./completions/checkShouldExtract-f0";
import { batchExtractPromise_F0 } from "./completions/batchExtract-f0";
@@ -250,7 +250,7 @@ export async function performExtraction_F0(
}
if (reqSchema) {
- reqSchema = await dereferenceSchema_F0(reqSchema);
+ reqSchema = await dereferenceSchema(reqSchema);
}
logger.debug("Transformed schema.", {
diff --git a/apps/api/src/lib/extract/fire-0/helpers/dereference-schema-f0.ts b/apps/api/src/lib/extract/fire-0/helpers/dereference-schema-f0.ts
deleted file mode 100644
index d4f863a7b..000000000
--- a/apps/api/src/lib/extract/fire-0/helpers/dereference-schema-f0.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import { dereference } from "@apidevtools/json-schema-ref-parser";
-
-export async function dereferenceSchema_F0(schema: any): Promise<any> {
- try {
- return await dereference(schema);
- } catch (error) {
- console.error("Failed to dereference schema:", error);
- throw error;
- }
-}
diff --git a/apps/api/src/lib/extract/helpers/__tests__/dereference-schema.test.ts b/apps/api/src/lib/extract/helpers/__tests__/dereference-schema.test.ts
new file mode 100644
index 000000000..7730d56c2
--- /dev/null
+++ b/apps/api/src/lib/extract/helpers/__tests__/dereference-schema.test.ts
@@ -0,0 +1,105 @@
+import { dereferenceSchema } from "../dereference-schema";
+
+describe("dereferenceSchema", () => {
+ it("inlines an internal $ref pointer", async () => {
+ const schema = {
+ type: "object",
+ properties: {
+ user: { $ref: "#/definitions/User" },
+ },
+ definitions: {
+ User: { type: "object", properties: { name: { type: "string" } } },
+ },
+ };
+
+ const result = await dereferenceSchema(schema);
+
+ expect(result.properties.user).toEqual({
+ type: "object",
+ properties: { name: { type: "string" } },
+ });
+ });
+
+ it("resolves nested refs (a ref target that itself contains a ref)", async () => {
+ const schema = {
+ properties: { a: { $ref: "#/definitions/A" } },
+ definitions: {
+ A: { properties: { b: { $ref: "#/definitions/B" } } },
+ B: { type: "number" },
+ },
+ };
+
+ const result = await dereferenceSchema(schema);
+
+ expect(result.properties.a.properties.b).toEqual({ type: "number" });
+ });
+
+ it("does not mutate the input schema", async () => {
+ const schema = {
+ properties: { user: { $ref: "#/definitions/User" } },
+ definitions: { User: { type: "string" } },
+ };
+
+ await dereferenceSchema(schema);
+
+ expect(schema.properties.user).toEqual({ $ref: "#/definitions/User" });
+ });
+
+ it("breaks self-referential cycles without infinite recursion", async () => {
+ const schema = {
+ definitions: {
+ Node: {
+ type: "object",
+ properties: { next: { $ref: "#/definitions/Node" } },
+ },
+ },
+ properties: { root: { $ref: "#/definitions/Node" } },
+ };
+
+ const result = await dereferenceSchema(schema);
+
+ // One level is inlined; the cycle back to Node is left as a $ref.
+ expect(result.properties.root.type).toBe("object");
+ expect(result.properties.root.properties.next).toEqual({
+ $ref: "#/definitions/Node",
+ });
+ });
+
+ // Security: the whole reason this resolver exists. A $ref must NEVER be
+ // resolved against the filesystem or network — only inside the document.
+ it("leaves a file:// $ref untouched and reads nothing", async () => {
+ const schema = {
+ type: "object",
+ properties: {
+ x: { default: { $ref: "/etc/passwd#/nonexistent" } },
+ },
+ };
+
+ const result = await dereferenceSchema(schema);
+
+ // The external ref is passed through verbatim, not resolved to file contents.
+ expect(result.properties.x.default).toEqual({
+ $ref: "/etc/passwd#/nonexistent",
+ });
+ });
+
+ it("leaves an http(s):// $ref untouched (no SSRF)", async () => {
+ const schema = {
+ properties: { y: { $ref: "http://169.254.169.254/latest/meta-data/" } },
+ };
+
+ const result = await dereferenceSchema(schema);
+
+ expect(result.properties.y).toEqual({
+ $ref: "http://169.254.169.254/latest/meta-data/",
+ });
+ });
+
+ it("leaves an unresolvable internal pointer as-is", async () => {
+ const schema = { properties: { z: { $ref: "#/definitions/Missing" } } };
+
+ const result = await dereferenceSchema(schema);
+
+ expect(result.properties.z).toEqual({ $ref: "#/definitions/Missing" });
+ });
+});
diff --git a/apps/api/src/lib/extract/helpers/dereference-schema.ts b/apps/api/src/lib/extract/helpers/dereference-schema.ts
index 66f8fa504..895d83012 100644
--- a/apps/api/src/lib/extract/helpers/dereference-schema.ts
+++ b/apps/api/src/lib/extract/helpers/dereference-schema.ts
@@ -1,8 +1,73 @@
-import { dereference } from "@apidevtools/json-schema-ref-parser";
+const MAX_REF_RESOLUTIONS = 50_000;
+
+function unescapePointerSegment(segment: string): string {
+ // JSON Pointer escaping: "~1" -> "/", "~0" -> "~" (order matters).
+ return segment.replace(/~1/g, "/").replace(/~0/g, "~");
+}
+
+function resolveJsonPointer(root: any, ref: string): any {
+ if (ref === "#") return root;
+ // Only plain JSON-pointer fragments are supported. Anchor refs ("#foo") and
+ // anything else are deliberately left unresolved rather than guessed at.
+ if (!ref.startsWith("#/")) return undefined;
+
+ let current = root;
+ for (const segment of ref.slice(2).split("/").map(unescapePointerSegment)) {
+ if (current === null || typeof current !== "object") return undefined;
+ current = current[segment];
+ if (current === undefined) return undefined;
+ }
+ return current;
+}
+
+function isRefObject(node: any): node is { $ref: string } {
+ return (
+ node !== null &&
+ typeof node === "object" &&
+ !Array.isArray(node) &&
+ typeof node.$ref === "string"
+ );
+}
export async function dereferenceSchema(schema: any): Promise<any> {
+ const root = schema;
+ let resolutions = 0;
+
+ // `activeRefs` holds the pointers currently being expanded along this path;
+ // revisiting one means a cycle, which we break by leaving the $ref in place.
+ function walk(node: any, activeRefs: Set<string>): any {
+ if (Array.isArray(node)) {
+ return node.map(item => walk(item, activeRefs));
+ }
+ if (node === null || typeof node !== "object") {
+ return node;
+ }
+ if (isRefObject(node)) {
+ const ref = node.$ref;
+ // External ref, cycle, or unresolvable pointer: leave the node as-is.
+ if (!ref.startsWith("#") || activeRefs.has(ref)) {
+ return { ...node };
+ }
+ const target = resolveJsonPointer(root, ref);
+ if (target === undefined) {
+ return { ...node };
+ }
+ if (++resolutions > MAX_REF_RESOLUTIONS) {
+ throw new Error("Schema $ref resolution limit exceeded");
+ }
+ const nextActive = new Set(activeRefs);
+ nextActive.add(ref);
+ return walk(target, nextActive);
+ }
+ const result: Record<string, any> = {};
+ for (const key of Object.keys(node)) {
+ result[key] = walk(node[key], activeRefs);
+ }
+ return result;
+ }
+
try {
- return await dereference(schema);
+ return walk(root, new Set());
} catch (error) {
console.error("Failed to dereference schema:", error);
throw error;