Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: Add Zod validation to reject additionalProperties in schema parameters (#2058)
devin-ai-integration[bot]1f7dfbf573c0
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.tsindex d6f95edf7..972964d81 100644--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts@@ -7,6 +7,7 @@ import {idmux,Identity,scrapeRaw,+ extract,} from "./lib";import crypto from "crypto";@@ -1574,4 +1575,152 @@ describe("Scrape tests", () => {scrapeTimeout,);});++ describe("Schema validation for additionalProperties", () => {+ if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {+ it("should normalize scrape request with additionalProperties in extract schema", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ properties: {+ title: { type: "string" }+ },+ additionalProperties: false+ }+ }+ }, identity);++ expect(response.statusCode).toBe(200);+ }, scrapeTimeout);++ it("should normalize extract request with additionalProperties in schema", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await extract({+ urls: ["https://example.com"],+ schema: {+ type: "object",+ properties: {+ title: { type: "string" }+ },+ additionalProperties: true+ }+ }, identity);++ expect(response.success).toBe(true);+ }, scrapeTimeout);++ it("should normalize scrape request with nested additionalProperties", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ properties: {+ user: {+ type: "object",+ properties: {+ name: { type: "string" }+ },+ additionalProperties: false+ }+ }+ }+ }+ }, identity);++ expect(response.statusCode).toBe(200);+ }, scrapeTimeout);++ it("should accept valid schema without additionalProperties", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ properties: {+ title: { type: "string" }+ },+ required: ["title"]+ }+ }+ }, identity);++ expect(response.statusCode).toBe(200);+ }, scrapeTimeout);++ it("should normalize changeTracking with additionalProperties in schema", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: ["markdown", "changeTracking"],+ changeTrackingOptions: {+ schema: {+ type: "object",+ properties: {+ changes: { type: "string" }+ },+ additionalProperties: true+ }+ }+ }, identity);++ expect(response.statusCode).toBe(200);+ }, scrapeTimeout);++ it("should reject schema-less dictionary (no properties but additionalProperties: true)", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ additionalProperties: true+ }+ }+ }, identity);++ expect(response.statusCode).toBe(400);+ expect(response.body.error).toContain("OpenAI");+ expect(response.body.error).toContain("schema-less dictionary");+ }, scrapeTimeout);++ it("should normalize scrape request with object type without properties (but no additionalProperties)", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ properties: {+ address: { type: "string" },+ detail: {+ type: "object",+ description: "Any other specifications of the particular make and model in the page"+ }+ }+ }+ }+ }, identity);++ expect(response.statusCode).toBe(200);+ }, scrapeTimeout);+ }+ });});diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex 6ab7c96f1..1637c1468 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -472,6 +472,67 @@ export async function batchScrapeWithConcurrencyTracking(};}+// =========================================+// Extract API+// =========================================++async function extractStart(body: ExtractRequestInput, identity: Identity) {+ return await request(TEST_URL)+ .post("/v2/extract")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body);+}++async function extractStatus(id: string, identity: Identity) {+ return await request(TEST_URL)+ .get("/v2/extract/" + encodeURIComponent(id))+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .send();+}++function expectExtractStartToSucceed(+ response: Awaited<ReturnType<typeof extractStart>>,+) {+ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(typeof response.body.id).toBe("string");+}++function expectExtractToSucceed(+ response: Awaited<ReturnType<typeof extractStatus>>,+) {+ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(typeof response.body.status).toBe("string");+ expect(response.body.status).toBe("completed");+ expect(response.body).toHaveProperty("data");+}++export async function extract(+ body: ExtractRequestInput,+ identity: Identity,+): Promise<ExtractResponse> {+ const es = await extractStart(body, identity);+ expectExtractStartToSucceed(es);++ let x;++ do {+ x = await extractStatus(es.body.id, identity);+ expect(x.statusCode).toBe(200);+ expect(typeof x.body.status).toBe("string");+ } while (x.body.status === "processing");++ expectExtractToSucceed(x);+ return x.body;+}++export async function extractRaw(body: ExtractRequestInput, identity: Identity) {+ return await extractStart(body, identity);+}++// =========================================// ZDR API// =========================================diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 2f6f57cc6..d20f9a727 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -7,6 +7,7 @@ import {idmux,Identity,scrapeRaw,+ extractRaw,} from "./lib";import crypto from "crypto";@@ -1714,4 +1715,138 @@ describe("attributes format", () => {},scrapeTimeout,);++ describe("Schema validation for additionalProperties", () => {+ if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {+ it("should normalize scrape request with additionalProperties in json format schema", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: {+ title: { type: "string" }+ },+ additionalProperties: false+ }+ }+ ]+ }, identity);++ expect(response.statusCode).toBe(200);+ }, scrapeTimeout);++ it("should normalize extract request with additionalProperties in schema", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await extractRaw({+ urls: ["https://example.com"],+ schema: {+ type: "object",+ properties: {+ title: { type: "string" }+ },+ additionalProperties: true+ }+ }, identity);++ expect(response.statusCode).toBe(200);+ }, scrapeTimeout);++ it("should normalize changeTracking format with additionalProperties", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: [+ { type: "markdown" },+ {+ type: "changeTracking",+ schema: {+ type: "object",+ properties: {+ changes: { type: "string" }+ },+ additionalProperties: false+ }+ }+ ]+ }, identity);++ expect(response.statusCode).toBe(200);+ }, scrapeTimeout);++ it("should accept valid schema without additionalProperties", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: {+ title: { type: "string" }+ },+ required: ["title"]+ }+ }+ ]+ }, identity);++ expect(response.statusCode).toBe(200);+ }, scrapeTimeout);++ it("should reject schema-less dictionary (no properties but additionalProperties: true)", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ additionalProperties: true+ }+ }+ ]+ }, identity);++ expect(response.statusCode).toBe(400);+ expect(response.body.error).toContain("OpenAI");+ expect(response.body.error).toContain("schema-less dictionary");+ }, scrapeTimeout);++ it("should normalize schema with object type without properties (but no additionalProperties)", async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw({+ url: "https://example.com",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: {+ address: { type: "string" },+ detail: {+ type: "object",+ description: "Any other specifications of the particular make and model in the page"+ }+ }+ }+ }+ ]+ }, identity);++ expect(response.statusCode).toBe(200);+ }, scrapeTimeout);+ }+ });});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 683dea09e..c31517350 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -74,6 +74,81 @@ const agentExtractModelValue = "fire-1";export const isAgentExtractModelValid = (x: string | undefined) =>x?.toLowerCase() === agentExtractModelValue;+function normalizeSchemaForOpenAI(schema: any): any {+ if (!schema || typeof schema !== 'object') {+ return schema;+ }++ const visited = new WeakSet();++ function normalizeObject(obj: any): any {+ if (typeof obj !== 'object' || obj === null) return obj;++ if (visited.has(obj)) return obj;+ visited.add(obj);++ const normalized = { ...obj };++ if (normalized.type === 'object' && normalized.hasOwnProperty('properties') && normalized.hasOwnProperty('additionalProperties')) {+ delete normalized.additionalProperties;+ }++ if (normalized.type === 'object' && normalized.hasOwnProperty('required') && normalized.hasOwnProperty('properties')) {+ if (Array.isArray(normalized.required) && typeof normalized.properties === 'object' && normalized.properties !== null) {+ const validRequired = normalized.required.filter((field: string) =>+ normalized.properties.hasOwnProperty(field)+ );+ if (validRequired.length > 0) {+ normalized.required = validRequired;+ } else {+ delete normalized.required;+ }+ } else {+ delete normalized.required;+ }+ }++ for (const [key, value] of Object.entries(normalized)) {+ if (typeof value === 'object' && value !== null) {+ normalized[key] = normalizeObject(value);+ }+ }++ return normalized;+ }++ return normalizeObject(schema);+}+function validateSchemaForOpenAI(schema: any): boolean {+ if (!schema || typeof schema !== 'object') {+ return true;+ }++ const visited = new WeakSet();++ function hasInvalidStructure(obj: any): boolean {+ if (typeof obj !== 'object' || obj === null) return false;++ if (visited.has(obj)) return false;+ visited.add(obj);++ if (obj.type === 'object' && !obj.hasOwnProperty('properties') && !obj.hasOwnProperty('patternProperties') && obj.additionalProperties === true) {+ return true;+ }++ for (const value of Object.values(obj)) {+ if (typeof value === 'object' && value !== null) {+ if (hasInvalidStructure(value)) return true;+ }+ }+ return false;+ }++ return !hasInvalidStructure(schema);+}++const OPENAI_SCHEMA_ERROR_MESSAGE = "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";+export const agentOptionsExtract = z.object({model: z.string().default(agentExtractModelValue),@@ -99,6 +174,13 @@ export const extractOptions = z{message: "Invalid JSON schema.",},+ )+ .transform((val) => normalizeSchemaForOpenAI(val))+ .refine(+ (val) => validateSchemaForOpenAI(val),+ {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ },),systemPrompt: z.string().max(10000).default(""),prompt: z.string().max(10000).optional(),@@ -130,6 +212,13 @@ const extractOptionsWithAgent = z{message: "Invalid JSON schema.",},+ )+ .transform((val) => normalizeSchemaForOpenAI(val))+ .refine(+ (val) => validateSchemaForOpenAI(val),+ {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ },),systemPrompt: z.string().max(10000).default(""),prompt: z.string().max(10000).optional(),@@ -367,6 +456,13 @@ const baseScrapeOptions = z{message: "Invalid JSON schema.",},+ )+ .transform((val) => normalizeSchemaForOpenAI(val))+ .refine(+ (val) => validateSchemaForOpenAI(val),+ {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ },),modes: z.enum(["json", "git-diff"]).array().optional().default([]),tag: z.string().or(z.null()).default(null),@@ -573,6 +669,13 @@ const extractV1Options = z{message: "Invalid JSON schema.",},+ )+ .transform((val) => normalizeSchemaForOpenAI(val))+ .refine(+ (val) => validateSchemaForOpenAI(val),+ {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ },),limit: z.number().int().positive().finite().safe().optional(),ignoreSitemap: z.boolean().default(false),diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 998321b7e..fbc82e667 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -63,6 +63,82 @@ export const url = z.preprocess(const strictMessage ="Unrecognized key in body -- please review the v2 API documentation for request body changes";+function normalizeSchemaForOpenAI(schema: any): any {+ if (!schema || typeof schema !== 'object') {+ return schema;+ }++ const visited = new WeakSet();++ function normalizeObject(obj: any): any {+ if (typeof obj !== 'object' || obj === null) return obj;++ if (visited.has(obj)) return obj;+ visited.add(obj);++ const normalized = { ...obj };++ if (normalized.type === 'object' && normalized.hasOwnProperty('properties') && normalized.hasOwnProperty('additionalProperties')) {+ delete normalized.additionalProperties;+ }++ if (normalized.type === 'object' && normalized.hasOwnProperty('required') && normalized.hasOwnProperty('properties')) {+ if (Array.isArray(normalized.required) && typeof normalized.properties === 'object' && normalized.properties !== null) {+ const validRequired = normalized.required.filter((field: string) =>+ normalized.properties.hasOwnProperty(field)+ );+ if (validRequired.length > 0) {+ normalized.required = validRequired;+ } else {+ delete normalized.required;+ }+ } else {+ delete normalized.required;+ }+ }++ for (const [key, value] of Object.entries(normalized)) {+ if (typeof value === 'object' && value !== null) {+ normalized[key] = normalizeObject(value);+ }+ }++ return normalized;+ }++ return normalizeObject(schema);+}++function validateSchemaForOpenAI(schema: any): boolean {+ if (!schema || typeof schema !== 'object') {+ return true;+ }++ const visited = new WeakSet();++ function hasInvalidStructure(obj: any): boolean {+ if (typeof obj !== 'object' || obj === null) return false;++ if (visited.has(obj)) return false;+ visited.add(obj);++ if (obj.type === 'object' && !obj.hasOwnProperty('properties') && !obj.hasOwnProperty('patternProperties') && obj.additionalProperties === true) {+ return true;+ }++ for (const value of Object.values(obj)) {+ if (typeof value === 'object' && value !== null) {+ if (hasInvalidStructure(value)) return true;+ }+ }+ return false;+ }++ return !hasInvalidStructure(schema);+}++const OPENAI_SCHEMA_ERROR_MESSAGE = "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";+const ACTIONS_MAX_WAIT_TIME = 60;const MAX_ACTIONS = 50;function calculateTotalWaitTime(@@ -174,7 +250,14 @@ const actionsSchema = zconst jsonFormatWithOptions = z.object({type: z.literal("json"),- schema: z.any().optional(),+ schema: z.any().optional()+ .transform((val) => normalizeSchemaForOpenAI(val))+ .refine(+ (val) => validateSchemaForOpenAI(val),+ {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ },+ ),prompt: z.string().max(10000).optional(),}).strict();@@ -185,7 +268,14 @@ const changeTrackingFormatWithOptions = z.object({type: z.literal("changeTracking"),prompt: z.string().optional(),- schema: z.any().optional(),+ schema: z.any().optional()+ .transform((val) => normalizeSchemaForOpenAI(val))+ .refine(+ (val) => validateSchemaForOpenAI(val),+ {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ },+ ),modes: z.enum(["json", "git-diff"]).array().optional().default([]),tag: z.string().or(z.null()).default(null),})@@ -478,6 +568,13 @@ const extractOptions = z{message: "Invalid JSON schema.",},+ )+ .transform((val) => normalizeSchemaForOpenAI(val))+ .refine(+ (val) => validateSchemaForOpenAI(val),+ {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ },),limit: z.number().int().positive().finite().safe().optional(),ignoreSitemap: z.boolean().default(false),diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex e6b4854da..9920d243c 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -172,10 +172,14 @@ app.use(logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));}+ const customErrorMessage = err.errors.length > 0 && err.errors[0].code === "custom"+ ? err.errors[0].message+ : "Bad Request";+res.status(400).json({success: false,code: "BAD_REQUEST",- error: "Bad Request",+ error: customErrorMessage,details: err.errors,});} else {@@ -226,4 +230,4 @@ app.use(},);-logger.info(`Worker ${process.pid} started`);+logger.info(`Worker ${process.pid} started`);