Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "fix: Add Zod validation to reject additionalProperties in schema parameters (#2058)"
Gergő Móricz8232a53a2194
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.tsindex 972964d81..d6f95edf7 100644--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts@@ -7,7 +7,6 @@ import {idmux,Identity,scrapeRaw,- extract,} from "./lib";import crypto from "crypto";@@ -1575,152 +1574,4 @@ describe("Scrape tests", () => {scrapeTimeout,);});-- describe("Schema validation for additionalProperties", () => {- if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {- it("should normalize scrape request with additionalProperties in extract schema", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: ["extract"],- extract: {- schema: {- type: "object",- properties: {- title: { type: "string" }- },- additionalProperties: false- }- }- }, identity);-- expect(response.statusCode).toBe(200);- }, scrapeTimeout);-- it("should normalize extract request with additionalProperties in schema", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await extract({- urls: ["https://example.com"],- schema: {- type: "object",- properties: {- title: { type: "string" }- },- additionalProperties: true- }- }, identity);-- expect(response.success).toBe(true);- }, scrapeTimeout);-- it("should normalize scrape request with nested additionalProperties", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: ["extract"],- extract: {- schema: {- type: "object",- properties: {- user: {- type: "object",- properties: {- name: { type: "string" }- },- additionalProperties: false- }- }- }- }- }, identity);-- expect(response.statusCode).toBe(200);- }, scrapeTimeout);-- it("should accept valid schema without additionalProperties", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: ["extract"],- extract: {- schema: {- type: "object",- properties: {- title: { type: "string" }- },- required: ["title"]- }- }- }, identity);-- expect(response.statusCode).toBe(200);- }, scrapeTimeout);-- it("should normalize changeTracking with additionalProperties in schema", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: ["markdown", "changeTracking"],- changeTrackingOptions: {- schema: {- type: "object",- properties: {- changes: { type: "string" }- },- additionalProperties: true- }- }- }, identity);-- expect(response.statusCode).toBe(200);- }, scrapeTimeout);-- it("should reject schema-less dictionary (no properties but additionalProperties: true)", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: ["extract"],- extract: {- schema: {- type: "object",- additionalProperties: true- }- }- }, identity);-- expect(response.statusCode).toBe(400);- expect(response.body.error).toContain("OpenAI");- expect(response.body.error).toContain("schema-less dictionary");- }, scrapeTimeout);-- it("should normalize scrape request with object type without properties (but no additionalProperties)", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: ["extract"],- extract: {- schema: {- type: "object",- properties: {- address: { type: "string" },- detail: {- type: "object",- description: "Any other specifications of the particular make and model in the page"- }- }- }- }- }, identity);-- expect(response.statusCode).toBe(200);- }, scrapeTimeout);- }- });});diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex 1637c1468..6ab7c96f1 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -472,67 +472,6 @@ export async function batchScrapeWithConcurrencyTracking(};}-// =========================================-// Extract API-// =========================================--async function extractStart(body: ExtractRequestInput, identity: Identity) {- return await request(TEST_URL)- .post("/v2/extract")- .set("Authorization", `Bearer ${identity.apiKey}`)- .set("Content-Type", "application/json")- .send(body);-}--async function extractStatus(id: string, identity: Identity) {- return await request(TEST_URL)- .get("/v2/extract/" + encodeURIComponent(id))- .set("Authorization", `Bearer ${identity.apiKey}`)- .send();-}--function expectExtractStartToSucceed(- response: Awaited<ReturnType<typeof extractStart>>,-) {- expect(response.statusCode).toBe(200);- expect(response.body.success).toBe(true);- expect(typeof response.body.id).toBe("string");-}--function expectExtractToSucceed(- response: Awaited<ReturnType<typeof extractStatus>>,-) {- expect(response.statusCode).toBe(200);- expect(response.body.success).toBe(true);- expect(typeof response.body.status).toBe("string");- expect(response.body.status).toBe("completed");- expect(response.body).toHaveProperty("data");-}--export async function extract(- body: ExtractRequestInput,- identity: Identity,-): Promise<ExtractResponse> {- const es = await extractStart(body, identity);- expectExtractStartToSucceed(es);-- let x;-- do {- x = await extractStatus(es.body.id, identity);- expect(x.statusCode).toBe(200);- expect(typeof x.body.status).toBe("string");- } while (x.body.status === "processing");-- expectExtractToSucceed(x);- return x.body;-}--export async function extractRaw(body: ExtractRequestInput, identity: Identity) {- return await extractStart(body, identity);-}--// =========================================// ZDR API// =========================================diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex d20f9a727..2f6f57cc6 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -7,7 +7,6 @@ import {idmux,Identity,scrapeRaw,- extractRaw,} from "./lib";import crypto from "crypto";@@ -1715,138 +1714,4 @@ describe("attributes format", () => {},scrapeTimeout,);-- describe("Schema validation for additionalProperties", () => {- if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {- it("should normalize scrape request with additionalProperties in json format schema", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: [- {- type: "json",- schema: {- type: "object",- properties: {- title: { type: "string" }- },- additionalProperties: false- }- }- ]- }, identity);-- expect(response.statusCode).toBe(200);- }, scrapeTimeout);-- it("should normalize extract request with additionalProperties in schema", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await extractRaw({- urls: ["https://example.com"],- schema: {- type: "object",- properties: {- title: { type: "string" }- },- additionalProperties: true- }- }, identity);-- expect(response.statusCode).toBe(200);- }, scrapeTimeout);-- it("should normalize changeTracking format with additionalProperties", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: [- { type: "markdown" },- {- type: "changeTracking",- schema: {- type: "object",- properties: {- changes: { type: "string" }- },- additionalProperties: false- }- }- ]- }, identity);-- expect(response.statusCode).toBe(200);- }, scrapeTimeout);-- it("should accept valid schema without additionalProperties", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: [- {- type: "json",- schema: {- type: "object",- properties: {- title: { type: "string" }- },- required: ["title"]- }- }- ]- }, identity);-- expect(response.statusCode).toBe(200);- }, scrapeTimeout);-- it("should reject schema-less dictionary (no properties but additionalProperties: true)", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: [- {- type: "json",- schema: {- type: "object",- additionalProperties: true- }- }- ]- }, identity);-- expect(response.statusCode).toBe(400);- expect(response.body.error).toContain("OpenAI");- expect(response.body.error).toContain("schema-less dictionary");- }, scrapeTimeout);-- it("should normalize schema with object type without properties (but no additionalProperties)", async () => {- const identity = await idmux({ name: "schema-validation-test" });-- const response = await scrapeRaw({- url: "https://example.com",- formats: [- {- type: "json",- schema: {- type: "object",- properties: {- address: { type: "string" },- detail: {- type: "object",- description: "Any other specifications of the particular make and model in the page"- }- }- }- }- ]- }, identity);-- expect(response.statusCode).toBe(200);- }, scrapeTimeout);- }- });});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 0dc9999c7..b9f0419ef 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -74,99 +74,6 @@ const agentExtractModelValue = "fire-1";export const isAgentExtractModelValid = (x: string | undefined) =>x?.toLowerCase() === agentExtractModelValue;-function normalizeSchemaForOpenAI(schema: any): any {- if (!schema || typeof schema !== "object") {- return schema;- }-- const visited = new WeakSet();-- function normalizeObject(obj: any): any {- if (typeof obj !== "object" || obj === null) return obj;-- if (visited.has(obj)) return obj;- visited.add(obj);-- const normalized = { ...obj };-- if (- normalized.type === "object" &&- normalized.hasOwnProperty("properties") &&- normalized.hasOwnProperty("additionalProperties")- ) {- delete normalized.additionalProperties;- }-- if (- normalized.type === "object" &&- normalized.hasOwnProperty("required") &&- normalized.hasOwnProperty("properties")- ) {- if (- Array.isArray(normalized.required) &&- typeof normalized.properties === "object" &&- normalized.properties !== null- ) {- const validRequired = normalized.required.filter((field: string) =>- normalized.properties.hasOwnProperty(field),- );- if (validRequired.length > 0) {- normalized.required = validRequired;- } else {- delete normalized.required;- }- } else {- delete normalized.required;- }- }-- for (const [key, value] of Object.entries(normalized)) {- if (typeof value === "object" && value !== null) {- normalized[key] = normalizeObject(value);- }- }-- return normalized;- }-- return normalizeObject(schema);-}-function validateSchemaForOpenAI(schema: any): boolean {- if (!schema || typeof schema !== "object") {- return true;- }-- const visited = new WeakSet();-- function hasInvalidStructure(obj: any): boolean {- if (typeof obj !== "object" || obj === null) return false;-- if (visited.has(obj)) return false;- visited.add(obj);-- if (- obj.type === "object" &&- !obj.hasOwnProperty("properties") &&- !obj.hasOwnProperty("patternProperties") &&- obj.additionalProperties === true- ) {- return true;- }-- for (const value of Object.values(obj)) {- if (typeof value === "object" && value !== null) {- if (hasInvalidStructure(value)) return true;- }- }- return false;- }-- return !hasInvalidStructure(schema);-}--const OPENAI_SCHEMA_ERROR_MESSAGE =- "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";-export const agentOptionsExtract = z.object({model: z.string().default(agentExtractModelValue),@@ -192,11 +99,7 @@ export const extractOptions = z{message: "Invalid JSON schema.",},- )- .transform(val => normalizeSchemaForOpenAI(val))- .refine(val => validateSchemaForOpenAI(val), {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ ),systemPrompt: z.string().max(10000).default(""),prompt: z.string().max(10000).optional(),temperature: z.number().optional(),@@ -227,11 +130,7 @@ const extractOptionsWithAgent = z{message: "Invalid JSON schema.",},- )- .transform(val => normalizeSchemaForOpenAI(val))- .refine(val => validateSchemaForOpenAI(val), {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ ),systemPrompt: z.string().max(10000).default(""),prompt: z.string().max(10000).optional(),temperature: z.number().optional(),@@ -468,11 +367,7 @@ const baseScrapeOptions = z{message: "Invalid JSON schema.",},- )- .transform(val => normalizeSchemaForOpenAI(val))- .refine(val => validateSchemaForOpenAI(val), {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ ),modes: z.enum(["json", "git-diff"]).array().optional().default([]),tag: z.string().or(z.null()).default(null),})@@ -678,11 +573,7 @@ const extractV1Options = z{message: "Invalid JSON schema.",},- )- .transform(val => normalizeSchemaForOpenAI(val))- .refine(val => validateSchemaForOpenAI(val), {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ ),limit: z.number().int().positive().finite().safe().optional(),ignoreSitemap: z.boolean().default(false),includeSubdomains: z.boolean().default(true),diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 197b86b73..167299396 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -63,100 +63,6 @@ export const url = z.preprocess(const strictMessage ="Unrecognized key in body -- please review the v2 API documentation for request body changes";-function normalizeSchemaForOpenAI(schema: any): any {- if (!schema || typeof schema !== "object") {- return schema;- }-- const visited = new WeakSet();-- function normalizeObject(obj: any): any {- if (typeof obj !== "object" || obj === null) return obj;-- if (visited.has(obj)) return obj;- visited.add(obj);-- const normalized = { ...obj };-- if (- normalized.type === "object" &&- normalized.hasOwnProperty("properties") &&- normalized.hasOwnProperty("additionalProperties")- ) {- delete normalized.additionalProperties;- }-- if (- normalized.type === "object" &&- normalized.hasOwnProperty("required") &&- normalized.hasOwnProperty("properties")- ) {- if (- Array.isArray(normalized.required) &&- typeof normalized.properties === "object" &&- normalized.properties !== null- ) {- const validRequired = normalized.required.filter((field: string) =>- normalized.properties.hasOwnProperty(field),- );- if (validRequired.length > 0) {- normalized.required = validRequired;- } else {- delete normalized.required;- }- } else {- delete normalized.required;- }- }-- for (const [key, value] of Object.entries(normalized)) {- if (typeof value === "object" && value !== null) {- normalized[key] = normalizeObject(value);- }- }-- return normalized;- }-- return normalizeObject(schema);-}--function validateSchemaForOpenAI(schema: any): boolean {- if (!schema || typeof schema !== "object") {- return true;- }-- const visited = new WeakSet();-- function hasInvalidStructure(obj: any): boolean {- if (typeof obj !== "object" || obj === null) return false;-- if (visited.has(obj)) return false;- visited.add(obj);-- if (- obj.type === "object" &&- !obj.hasOwnProperty("properties") &&- !obj.hasOwnProperty("patternProperties") &&- obj.additionalProperties === true- ) {- return true;- }-- for (const value of Object.values(obj)) {- if (typeof value === "object" && value !== null) {- if (hasInvalidStructure(value)) return true;- }- }- return false;- }-- return !hasInvalidStructure(schema);-}--const OPENAI_SCHEMA_ERROR_MESSAGE =- "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";-const ACTIONS_MAX_WAIT_TIME = 60;const MAX_ACTIONS = 50;function calculateTotalWaitTime(@@ -268,13 +174,7 @@ const actionsSchema = zconst jsonFormatWithOptions = z.object({type: z.literal("json"),- schema: z- .any()- .optional()- .transform(val => normalizeSchemaForOpenAI(val))- .refine(val => validateSchemaForOpenAI(val), {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ schema: z.any().optional(),prompt: z.string().max(10000).optional(),}).strict();@@ -285,13 +185,7 @@ const changeTrackingFormatWithOptions = z.object({type: z.literal("changeTracking"),prompt: z.string().optional(),- schema: z- .any()- .optional()- .transform(val => normalizeSchemaForOpenAI(val))- .refine(val => validateSchemaForOpenAI(val), {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ schema: z.any().optional(),modes: z.enum(["json", "git-diff"]).array().optional().default([]),tag: z.string().or(z.null()).default(null),})@@ -584,11 +478,7 @@ const extractOptions = z{message: "Invalid JSON schema.",},- )- .transform(val => normalizeSchemaForOpenAI(val))- .refine(val => validateSchemaForOpenAI(val), {- message: OPENAI_SCHEMA_ERROR_MESSAGE,- }),+ ),limit: z.number().int().positive().finite().safe().optional(),ignoreSitemap: z.boolean().default(false),includeSubdomains: z.boolean().default(true),diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 9920d243c..e6b4854da 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -172,14 +172,10 @@ app.use(logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));}- const customErrorMessage = err.errors.length > 0 && err.errors[0].code === "custom"- ? err.errors[0].message- : "Bad Request";-res.status(400).json({success: false,code: "BAD_REQUEST",- error: customErrorMessage,+ error: "Bad Request",details: err.errors,});} else {@@ -230,4 +226,4 @@ app.use(},);-logger.info(`Worker ${process.pid} started`);+logger.info(`Worker ${process.pid} started`);