Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "fix: Add Zod validation to reject additionalProperties in schema parameters (#2058)"

Gergő Móricz8232a53a2194
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.ts
index 972964d81..d6f95edf7 100644
--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts
@@ -7,7 +7,6 @@ import {
idmux,
Identity,
scrapeRaw,
- extract,
} from "./lib";
import crypto from "crypto";
@@ -1575,152 +1574,4 @@ describe("Scrape tests", () => {
scrapeTimeout,
);
});
-
- describe("Schema validation for additionalProperties", () => {
- if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {
- it("should normalize scrape request with additionalProperties in extract schema", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: ["extract"],
- extract: {
- schema: {
- type: "object",
- properties: {
- title: { type: "string" }
- },
- additionalProperties: false
- }
- }
- }, identity);
-
- expect(response.statusCode).toBe(200);
- }, scrapeTimeout);
-
- it("should normalize extract request with additionalProperties in schema", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await extract({
- urls: ["https://example.com"],
- schema: {
- type: "object",
- properties: {
- title: { type: "string" }
- },
- additionalProperties: true
- }
- }, identity);
-
- expect(response.success).toBe(true);
- }, scrapeTimeout);
-
- it("should normalize scrape request with nested additionalProperties", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: ["extract"],
- extract: {
- schema: {
- type: "object",
- properties: {
- user: {
- type: "object",
- properties: {
- name: { type: "string" }
- },
- additionalProperties: false
- }
- }
- }
- }
- }, identity);
-
- expect(response.statusCode).toBe(200);
- }, scrapeTimeout);
-
- it("should accept valid schema without additionalProperties", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: ["extract"],
- extract: {
- schema: {
- type: "object",
- properties: {
- title: { type: "string" }
- },
- required: ["title"]
- }
- }
- }, identity);
-
- expect(response.statusCode).toBe(200);
- }, scrapeTimeout);
-
- it("should normalize changeTracking with additionalProperties in schema", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: ["markdown", "changeTracking"],
- changeTrackingOptions: {
- schema: {
- type: "object",
- properties: {
- changes: { type: "string" }
- },
- additionalProperties: true
- }
- }
- }, identity);
-
- expect(response.statusCode).toBe(200);
- }, scrapeTimeout);
-
- it("should reject schema-less dictionary (no properties but additionalProperties: true)", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: ["extract"],
- extract: {
- schema: {
- type: "object",
- additionalProperties: true
- }
- }
- }, identity);
-
- expect(response.statusCode).toBe(400);
- expect(response.body.error).toContain("OpenAI");
- expect(response.body.error).toContain("schema-less dictionary");
- }, scrapeTimeout);
-
- it("should normalize scrape request with object type without properties (but no additionalProperties)", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: ["extract"],
- extract: {
- schema: {
- type: "object",
- properties: {
- address: { type: "string" },
- detail: {
- type: "object",
- description: "Any other specifications of the particular make and model in the page"
- }
- }
- }
- }
- }, identity);
-
- expect(response.statusCode).toBe(200);
- }, scrapeTimeout);
- }
- });
});
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts
index 1637c1468..6ab7c96f1 100644
--- a/apps/api/src/__tests__/snips/v2/lib.ts
+++ b/apps/api/src/__tests__/snips/v2/lib.ts
@@ -472,67 +472,6 @@ export async function batchScrapeWithConcurrencyTracking(
};
}
-// =========================================
-// Extract API
-// =========================================
-
-async function extractStart(body: ExtractRequestInput, identity: Identity) {
- return await request(TEST_URL)
- .post("/v2/extract")
- .set("Authorization", `Bearer ${identity.apiKey}`)
- .set("Content-Type", "application/json")
- .send(body);
-}
-
-async function extractStatus(id: string, identity: Identity) {
- return await request(TEST_URL)
- .get("/v2/extract/" + encodeURIComponent(id))
- .set("Authorization", `Bearer ${identity.apiKey}`)
- .send();
-}
-
-function expectExtractStartToSucceed(
- response: Awaited<ReturnType<typeof extractStart>>,
-) {
- expect(response.statusCode).toBe(200);
- expect(response.body.success).toBe(true);
- expect(typeof response.body.id).toBe("string");
-}
-
-function expectExtractToSucceed(
- response: Awaited<ReturnType<typeof extractStatus>>,
-) {
- expect(response.statusCode).toBe(200);
- expect(response.body.success).toBe(true);
- expect(typeof response.body.status).toBe("string");
- expect(response.body.status).toBe("completed");
- expect(response.body).toHaveProperty("data");
-}
-
-export async function extract(
- body: ExtractRequestInput,
- identity: Identity,
-): Promise<ExtractResponse> {
- const es = await extractStart(body, identity);
- expectExtractStartToSucceed(es);
-
- let x;
-
- do {
- x = await extractStatus(es.body.id, identity);
- expect(x.statusCode).toBe(200);
- expect(typeof x.body.status).toBe("string");
- } while (x.body.status === "processing");
-
- expectExtractToSucceed(x);
- return x.body;
-}
-
-export async function extractRaw(body: ExtractRequestInput, identity: Identity) {
- return await extractStart(body, identity);
-}
-
-
// =========================================
// ZDR API
// =========================================
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index d20f9a727..2f6f57cc6 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -7,7 +7,6 @@ import {
idmux,
Identity,
scrapeRaw,
- extractRaw,
} from "./lib";
import crypto from "crypto";
@@ -1715,138 +1714,4 @@ describe("attributes format", () => {
},
scrapeTimeout,
);
-
- describe("Schema validation for additionalProperties", () => {
- if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {
- it("should normalize scrape request with additionalProperties in json format schema", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: [
- {
- type: "json",
- schema: {
- type: "object",
- properties: {
- title: { type: "string" }
- },
- additionalProperties: false
- }
- }
- ]
- }, identity);
-
- expect(response.statusCode).toBe(200);
- }, scrapeTimeout);
-
- it("should normalize extract request with additionalProperties in schema", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await extractRaw({
- urls: ["https://example.com"],
- schema: {
- type: "object",
- properties: {
- title: { type: "string" }
- },
- additionalProperties: true
- }
- }, identity);
-
- expect(response.statusCode).toBe(200);
- }, scrapeTimeout);
-
- it("should normalize changeTracking format with additionalProperties", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: [
- { type: "markdown" },
- {
- type: "changeTracking",
- schema: {
- type: "object",
- properties: {
- changes: { type: "string" }
- },
- additionalProperties: false
- }
- }
- ]
- }, identity);
-
- expect(response.statusCode).toBe(200);
- }, scrapeTimeout);
-
- it("should accept valid schema without additionalProperties", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: [
- {
- type: "json",
- schema: {
- type: "object",
- properties: {
- title: { type: "string" }
- },
- required: ["title"]
- }
- }
- ]
- }, identity);
-
- expect(response.statusCode).toBe(200);
- }, scrapeTimeout);
-
- it("should reject schema-less dictionary (no properties but additionalProperties: true)", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: [
- {
- type: "json",
- schema: {
- type: "object",
- additionalProperties: true
- }
- }
- ]
- }, identity);
-
- expect(response.statusCode).toBe(400);
- expect(response.body.error).toContain("OpenAI");
- expect(response.body.error).toContain("schema-less dictionary");
- }, scrapeTimeout);
-
- it("should normalize schema with object type without properties (but no additionalProperties)", async () => {
- const identity = await idmux({ name: "schema-validation-test" });
-
- const response = await scrapeRaw({
- url: "https://example.com",
- formats: [
- {
- type: "json",
- schema: {
- type: "object",
- properties: {
- address: { type: "string" },
- detail: {
- type: "object",
- description: "Any other specifications of the particular make and model in the page"
- }
- }
- }
- }
- ]
- }, identity);
-
- expect(response.statusCode).toBe(200);
- }, scrapeTimeout);
- }
- });
});
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 0dc9999c7..b9f0419ef 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -74,99 +74,6 @@ const agentExtractModelValue = "fire-1";
export const isAgentExtractModelValid = (x: string | undefined) =>
x?.toLowerCase() === agentExtractModelValue;
-function normalizeSchemaForOpenAI(schema: any): any {
- if (!schema || typeof schema !== "object") {
- return schema;
- }
-
- const visited = new WeakSet();
-
- function normalizeObject(obj: any): any {
- if (typeof obj !== "object" || obj === null) return obj;
-
- if (visited.has(obj)) return obj;
- visited.add(obj);
-
- const normalized = { ...obj };
-
- if (
- normalized.type === "object" &&
- normalized.hasOwnProperty("properties") &&
- normalized.hasOwnProperty("additionalProperties")
- ) {
- delete normalized.additionalProperties;
- }
-
- if (
- normalized.type === "object" &&
- normalized.hasOwnProperty("required") &&
- normalized.hasOwnProperty("properties")
- ) {
- if (
- Array.isArray(normalized.required) &&
- typeof normalized.properties === "object" &&
- normalized.properties !== null
- ) {
- const validRequired = normalized.required.filter((field: string) =>
- normalized.properties.hasOwnProperty(field),
- );
- if (validRequired.length > 0) {
- normalized.required = validRequired;
- } else {
- delete normalized.required;
- }
- } else {
- delete normalized.required;
- }
- }
-
- for (const [key, value] of Object.entries(normalized)) {
- if (typeof value === "object" && value !== null) {
- normalized[key] = normalizeObject(value);
- }
- }
-
- return normalized;
- }
-
- return normalizeObject(schema);
-}
-function validateSchemaForOpenAI(schema: any): boolean {
- if (!schema || typeof schema !== "object") {
- return true;
- }
-
- const visited = new WeakSet();
-
- function hasInvalidStructure(obj: any): boolean {
- if (typeof obj !== "object" || obj === null) return false;
-
- if (visited.has(obj)) return false;
- visited.add(obj);
-
- if (
- obj.type === "object" &&
- !obj.hasOwnProperty("properties") &&
- !obj.hasOwnProperty("patternProperties") &&
- obj.additionalProperties === true
- ) {
- return true;
- }
-
- for (const value of Object.values(obj)) {
- if (typeof value === "object" && value !== null) {
- if (hasInvalidStructure(value)) return true;
- }
- }
- return false;
- }
-
- return !hasInvalidStructure(schema);
-}
-
-const OPENAI_SCHEMA_ERROR_MESSAGE =
- "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";
-
export const agentOptionsExtract = z
.object({
model: z.string().default(agentExtractModelValue),
@@ -192,11 +99,7 @@ export const extractOptions = z
{
message: "Invalid JSON schema.",
},
- )
- .transform(val => normalizeSchemaForOpenAI(val))
- .refine(val => validateSchemaForOpenAI(val), {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ ),
systemPrompt: z.string().max(10000).default(""),
prompt: z.string().max(10000).optional(),
temperature: z.number().optional(),
@@ -227,11 +130,7 @@ const extractOptionsWithAgent = z
{
message: "Invalid JSON schema.",
},
- )
- .transform(val => normalizeSchemaForOpenAI(val))
- .refine(val => validateSchemaForOpenAI(val), {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ ),
systemPrompt: z.string().max(10000).default(""),
prompt: z.string().max(10000).optional(),
temperature: z.number().optional(),
@@ -468,11 +367,7 @@ const baseScrapeOptions = z
{
message: "Invalid JSON schema.",
},
- )
- .transform(val => normalizeSchemaForOpenAI(val))
- .refine(val => validateSchemaForOpenAI(val), {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ ),
modes: z.enum(["json", "git-diff"]).array().optional().default([]),
tag: z.string().or(z.null()).default(null),
})
@@ -678,11 +573,7 @@ const extractV1Options = z
{
message: "Invalid JSON schema.",
},
- )
- .transform(val => normalizeSchemaForOpenAI(val))
- .refine(val => validateSchemaForOpenAI(val), {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ ),
limit: z.number().int().positive().finite().safe().optional(),
ignoreSitemap: z.boolean().default(false),
includeSubdomains: z.boolean().default(true),
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 197b86b73..167299396 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -63,100 +63,6 @@ export const url = z.preprocess(
const strictMessage =
"Unrecognized key in body -- please review the v2 API documentation for request body changes";
-function normalizeSchemaForOpenAI(schema: any): any {
- if (!schema || typeof schema !== "object") {
- return schema;
- }
-
- const visited = new WeakSet();
-
- function normalizeObject(obj: any): any {
- if (typeof obj !== "object" || obj === null) return obj;
-
- if (visited.has(obj)) return obj;
- visited.add(obj);
-
- const normalized = { ...obj };
-
- if (
- normalized.type === "object" &&
- normalized.hasOwnProperty("properties") &&
- normalized.hasOwnProperty("additionalProperties")
- ) {
- delete normalized.additionalProperties;
- }
-
- if (
- normalized.type === "object" &&
- normalized.hasOwnProperty("required") &&
- normalized.hasOwnProperty("properties")
- ) {
- if (
- Array.isArray(normalized.required) &&
- typeof normalized.properties === "object" &&
- normalized.properties !== null
- ) {
- const validRequired = normalized.required.filter((field: string) =>
- normalized.properties.hasOwnProperty(field),
- );
- if (validRequired.length > 0) {
- normalized.required = validRequired;
- } else {
- delete normalized.required;
- }
- } else {
- delete normalized.required;
- }
- }
-
- for (const [key, value] of Object.entries(normalized)) {
- if (typeof value === "object" && value !== null) {
- normalized[key] = normalizeObject(value);
- }
- }
-
- return normalized;
- }
-
- return normalizeObject(schema);
-}
-
-function validateSchemaForOpenAI(schema: any): boolean {
- if (!schema || typeof schema !== "object") {
- return true;
- }
-
- const visited = new WeakSet();
-
- function hasInvalidStructure(obj: any): boolean {
- if (typeof obj !== "object" || obj === null) return false;
-
- if (visited.has(obj)) return false;
- visited.add(obj);
-
- if (
- obj.type === "object" &&
- !obj.hasOwnProperty("properties") &&
- !obj.hasOwnProperty("patternProperties") &&
- obj.additionalProperties === true
- ) {
- return true;
- }
-
- for (const value of Object.values(obj)) {
- if (typeof value === "object" && value !== null) {
- if (hasInvalidStructure(value)) return true;
- }
- }
- return false;
- }
-
- return !hasInvalidStructure(schema);
-}
-
-const OPENAI_SCHEMA_ERROR_MESSAGE =
- "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";
-
const ACTIONS_MAX_WAIT_TIME = 60;
const MAX_ACTIONS = 50;
function calculateTotalWaitTime(
@@ -268,13 +174,7 @@ const actionsSchema = z
const jsonFormatWithOptions = z
.object({
type: z.literal("json"),
- schema: z
- .any()
- .optional()
- .transform(val => normalizeSchemaForOpenAI(val))
- .refine(val => validateSchemaForOpenAI(val), {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ schema: z.any().optional(),
prompt: z.string().max(10000).optional(),
})
.strict();
@@ -285,13 +185,7 @@ const changeTrackingFormatWithOptions = z
.object({
type: z.literal("changeTracking"),
prompt: z.string().optional(),
- schema: z
- .any()
- .optional()
- .transform(val => normalizeSchemaForOpenAI(val))
- .refine(val => validateSchemaForOpenAI(val), {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ schema: z.any().optional(),
modes: z.enum(["json", "git-diff"]).array().optional().default([]),
tag: z.string().or(z.null()).default(null),
})
@@ -584,11 +478,7 @@ const extractOptions = z
{
message: "Invalid JSON schema.",
},
- )
- .transform(val => normalizeSchemaForOpenAI(val))
- .refine(val => validateSchemaForOpenAI(val), {
- message: OPENAI_SCHEMA_ERROR_MESSAGE,
- }),
+ ),
limit: z.number().int().positive().finite().safe().optional(),
ignoreSitemap: z.boolean().default(false),
includeSubdomains: z.boolean().default(true),
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 9920d243c..e6b4854da 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -172,14 +172,10 @@ app.use(
logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));
}
- const customErrorMessage = err.errors.length > 0 && err.errors[0].code === "custom"
- ? err.errors[0].message
- : "Bad Request";
-
res.status(400).json({
success: false,
code: "BAD_REQUEST",
- error: customErrorMessage,
+ error: "Bad Request",
details: err.errors,
});
} else {
@@ -230,4 +226,4 @@ app.use(
},
);
-logger.info(`Worker ${process.pid} started`);
+logger.info(`Worker ${process.pid} started`);