Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: enum arrays being converted to objects (#2224)
Gaurav Chadhae8b5f562a0c4
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.tsindex d6f95edf7..5034bcc3f 100644--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts@@ -7,6 +7,7 @@ import {idmux,Identity,scrapeRaw,+ extract,} from "./lib";import crypto from "crypto";@@ -1574,4 +1575,206 @@ describe("Scrape tests", () => {scrapeTimeout,);});++ describe("Schema validation for additionalProperties", () => {+ if (+ !process.env.TEST_SUITE_SELF_HOSTED ||+ process.env.OPENAI_API_KEY ||+ process.env.OLLAMA_BASE_URL+ ) {+ it(+ "should normalize scrape request with additionalProperties in extract schema",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ properties: {+ title: { type: "string" },+ },+ additionalProperties: false,+ },+ },+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it(+ "should normalize extract request with additionalProperties in schema",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await extract(+ {+ urls: ["https://example.com"],+ schema: {+ type: "object",+ properties: {+ title: { type: "string" },+ },+ additionalProperties: true,+ },+ },+ identity,+ );++ expect(response.success).toBe(true);+ },+ scrapeTimeout,+ );++ it(+ "should normalize scrape request with nested additionalProperties",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ properties: {+ user: {+ type: "object",+ properties: {+ name: { type: "string" },+ },+ additionalProperties: false,+ },+ },+ },+ },+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it(+ "should accept valid schema without additionalProperties",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ properties: {+ title: { type: "string" },+ },+ required: ["title"],+ },+ },+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it(+ "should normalize changeTracking with additionalProperties in schema",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: ["markdown", "changeTracking"],+ changeTrackingOptions: {+ schema: {+ type: "object",+ properties: {+ changes: { type: "string" },+ },+ additionalProperties: true,+ },+ },+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it(+ "should reject schema-less dictionary (no properties but additionalProperties: true)",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ additionalProperties: true,+ },+ },+ },+ identity,+ );++ expect(response.statusCode).toBe(400);+ expect(response.body.error).toContain("OpenAI");+ expect(response.body.error).toContain("schema-less dictionary");+ },+ scrapeTimeout,+ );++ it(+ "should normalize scrape request with object type without properties (but no additionalProperties)",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: ["extract"],+ extract: {+ schema: {+ type: "object",+ properties: {+ address: { type: "string" },+ detail: {+ type: "object",+ description:+ "Any other specifications of the particular make and model in the page",+ },+ },+ },+ },+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ },+ scrapeTimeout,+ );+ }+ });});diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex 6ab7c96f1..47c4e1750 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -472,6 +472,69 @@ export async function batchScrapeWithConcurrencyTracking(};}+// =========================================+// Extract API+// =========================================++async function extractStart(body: ExtractRequestInput, identity: Identity) {+ return await request(TEST_URL)+ .post("/v2/extract")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body);+}++async function extractStatus(id: string, identity: Identity) {+ return await request(TEST_URL)+ .get("/v2/extract/" + encodeURIComponent(id))+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .send();+}++function expectExtractStartToSucceed(+ response: Awaited<ReturnType<typeof extractStart>>,+) {+ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(typeof response.body.id).toBe("string");+}++function expectExtractToSucceed(+ response: Awaited<ReturnType<typeof extractStatus>>,+) {+ expect(response.statusCode).toBe(200);+ expect(response.body.success).toBe(true);+ expect(typeof response.body.status).toBe("string");+ expect(response.body.status).toBe("completed");+ expect(response.body).toHaveProperty("data");+}++export async function extract(+ body: ExtractRequestInput,+ identity: Identity,+): Promise<ExtractResponse> {+ const es = await extractStart(body, identity);+ expectExtractStartToSucceed(es);++ let x;++ do {+ x = await extractStatus(es.body.id, identity);+ expect(x.statusCode).toBe(200);+ expect(typeof x.body.status).toBe("string");+ } while (x.body.status === "processing");++ expectExtractToSucceed(x);+ return x.body;+}++export async function extractRaw(+ body: ExtractRequestInput,+ identity: Identity,+) {+ return await extractStart(body, identity);+}+// =========================================// ZDR API// =========================================diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 2f6f57cc6..a1d20c658 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -7,6 +7,7 @@ import {idmux,Identity,scrapeRaw,+ extractRaw,} from "./lib";import crypto from "crypto";@@ -1714,4 +1715,185 @@ describe("attributes format", () => {},scrapeTimeout,);++ describe("Schema validation for additionalProperties", () => {+ if (+ !process.env.TEST_SUITE_SELF_HOSTED ||+ process.env.OPENAI_API_KEY ||+ process.env.OLLAMA_BASE_URL+ ) {+ it(+ "should normalize scrape request with additionalProperties in json format schema",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: {+ title: { type: "string" },+ },+ additionalProperties: false,+ },+ },+ ],+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it(+ "should normalize extract request with additionalProperties in schema",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await extractRaw(+ {+ urls: ["https://example.com"],+ schema: {+ type: "object",+ properties: {+ title: { type: "string" },+ },+ additionalProperties: true,+ },+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it(+ "should normalize changeTracking format with additionalProperties",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: [+ { type: "markdown" },+ {+ type: "changeTracking",+ schema: {+ type: "object",+ properties: {+ changes: { type: "string" },+ },+ additionalProperties: false,+ },+ },+ ],+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it(+ "should accept valid schema without additionalProperties",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: {+ title: { type: "string" },+ },+ required: ["title"],+ },+ },+ ],+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ },+ scrapeTimeout,+ );++ it(+ "should reject schema-less dictionary (no properties but additionalProperties: true)",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ additionalProperties: true,+ },+ },+ ],+ },+ identity,+ );++ expect(response.statusCode).toBe(400);+ expect(response.body.error).toContain("OpenAI");+ expect(response.body.error).toContain("schema-less dictionary");+ },+ scrapeTimeout,+ );++ it(+ "should normalize schema with object type without properties (but no additionalProperties)",+ async () => {+ const identity = await idmux({ name: "schema-validation-test" });++ const response = await scrapeRaw(+ {+ url: "https://example.com",+ formats: [+ {+ type: "json",+ schema: {+ type: "object",+ properties: {+ address: { type: "string" },+ detail: {+ type: "object",+ description:+ "Any other specifications of the particular make and model in the page",+ },+ },+ },+ },+ ],+ },+ identity,+ );++ expect(response.statusCode).toBe(200);+ },+ scrapeTimeout,+ );+ }+ });});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex b9f0419ef..1f7d8a4da 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -74,6 +74,100 @@ const agentExtractModelValue = "fire-1";export const isAgentExtractModelValid = (x: string | undefined) =>x?.toLowerCase() === agentExtractModelValue;+function normalizeSchemaForOpenAI(schema: any): any {+ if (!schema || typeof schema !== "object") {+ return schema;+ }++ const visited = new WeakSet();++ function normalizeObject(obj: any): any {+ if (typeof obj !== "object" || obj === null) return obj;+ if (Array.isArray(obj)) return obj;++ if (visited.has(obj)) return obj;+ visited.add(obj);++ const normalized = { ...obj };++ if (+ normalized.type === "object" &&+ normalized.hasOwnProperty("properties") &&+ normalized.hasOwnProperty("additionalProperties")+ ) {+ delete normalized.additionalProperties;+ }++ if (+ normalized.type === "object" &&+ normalized.hasOwnProperty("required") &&+ normalized.hasOwnProperty("properties")+ ) {+ if (+ Array.isArray(normalized.required) &&+ typeof normalized.properties === "object" &&+ normalized.properties !== null+ ) {+ const validRequired = normalized.required.filter((field: string) =>+ normalized.properties.hasOwnProperty(field),+ );+ if (validRequired.length > 0) {+ normalized.required = validRequired;+ } else {+ delete normalized.required;+ }+ } else {+ delete normalized.required;+ }+ }++ for (const [key, value] of Object.entries(normalized)) {+ if (typeof value === "object" && value !== null) {+ normalized[key] = normalizeObject(value);+ }+ }++ return normalized;+ }++ return normalizeObject(schema);+}+function validateSchemaForOpenAI(schema: any): boolean {+ if (!schema || typeof schema !== "object") {+ return true;+ }++ const visited = new WeakSet();++ function hasInvalidStructure(obj: any): boolean {+ if (typeof obj !== "object" || obj === null) return false;++ if (visited.has(obj)) return false;+ visited.add(obj);++ if (+ obj.type === "object" &&+ !obj.hasOwnProperty("properties") &&+ !obj.hasOwnProperty("patternProperties") &&+ obj.additionalProperties === true+ ) {+ return true;+ }++ for (const value of Object.values(obj)) {+ if (typeof value === "object" && value !== null) {+ if (hasInvalidStructure(value)) return true;+ }+ }+ return false;+ }++ return !hasInvalidStructure(schema);+}++const OPENAI_SCHEMA_ERROR_MESSAGE =+ "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";+export const agentOptionsExtract = z.object({model: z.string().default(agentExtractModelValue),@@ -99,7 +193,11 @@ export const extractOptions = z{message: "Invalid JSON schema.",},- ),+ )+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),systemPrompt: z.string().max(10000).default(""),prompt: z.string().max(10000).optional(),temperature: z.number().optional(),@@ -130,7 +228,11 @@ const extractOptionsWithAgent = z{message: "Invalid JSON schema.",},- ),+ )+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),systemPrompt: z.string().max(10000).default(""),prompt: z.string().max(10000).optional(),temperature: z.number().optional(),@@ -367,7 +469,11 @@ const baseScrapeOptions = z{message: "Invalid JSON schema.",},- ),+ )+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),modes: z.enum(["json", "git-diff"]).array().optional().default([]),tag: z.string().or(z.null()).default(null),})@@ -573,7 +679,11 @@ const extractV1Options = z{message: "Invalid JSON schema.",},- ),+ )+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),limit: z.number().int().positive().finite().safe().optional(),ignoreSitemap: z.boolean().default(false),includeSubdomains: z.boolean().default(true),diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 167299396..dd7c0c168 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -63,6 +63,101 @@ export const url = z.preprocess(const strictMessage ="Unrecognized key in body -- please review the v2 API documentation for request body changes";+function normalizeSchemaForOpenAI(schema: any): any {+ if (!schema || typeof schema !== "object") {+ return schema;+ }++ const visited = new WeakSet();++ function normalizeObject(obj: any): any {+ if (typeof obj !== "object" || obj === null) return obj;+ if (Array.isArray(obj)) return obj;++ if (visited.has(obj)) return obj;+ visited.add(obj);++ const normalized = { ...obj };++ if (+ normalized.type === "object" &&+ normalized.hasOwnProperty("properties") &&+ normalized.hasOwnProperty("additionalProperties")+ ) {+ delete normalized.additionalProperties;+ }++ if (+ normalized.type === "object" &&+ normalized.hasOwnProperty("required") &&+ normalized.hasOwnProperty("properties")+ ) {+ if (+ Array.isArray(normalized.required) &&+ typeof normalized.properties === "object" &&+ normalized.properties !== null+ ) {+ const validRequired = normalized.required.filter((field: string) =>+ normalized.properties.hasOwnProperty(field),+ );+ if (validRequired.length > 0) {+ normalized.required = validRequired;+ } else {+ delete normalized.required;+ }+ } else {+ delete normalized.required;+ }+ }++ for (const [key, value] of Object.entries(normalized)) {+ if (typeof value === "object" && value !== null) {+ normalized[key] = normalizeObject(value);+ }+ }++ return normalized;+ }++ return normalizeObject(schema);+}++function validateSchemaForOpenAI(schema: any): boolean {+ if (!schema || typeof schema !== "object") {+ return true;+ }++ const visited = new WeakSet();++ function hasInvalidStructure(obj: any): boolean {+ if (typeof obj !== "object" || obj === null) return false;++ if (visited.has(obj)) return false;+ visited.add(obj);++ if (+ obj.type === "object" &&+ !obj.hasOwnProperty("properties") &&+ !obj.hasOwnProperty("patternProperties") &&+ obj.additionalProperties === true+ ) {+ return true;+ }++ for (const value of Object.values(obj)) {+ if (typeof value === "object" && value !== null) {+ if (hasInvalidStructure(value)) return true;+ }+ }+ return false;+ }++ return !hasInvalidStructure(schema);+}++const OPENAI_SCHEMA_ERROR_MESSAGE =+ "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";+const ACTIONS_MAX_WAIT_TIME = 60;const MAX_ACTIONS = 50;function calculateTotalWaitTime(@@ -174,7 +269,13 @@ const actionsSchema = zconst jsonFormatWithOptions = z.object({type: z.literal("json"),- schema: z.any().optional(),+ schema: z+ .any()+ .optional()+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),prompt: z.string().max(10000).optional(),}).strict();@@ -185,7 +286,13 @@ const changeTrackingFormatWithOptions = z.object({type: z.literal("changeTracking"),prompt: z.string().optional(),- schema: z.any().optional(),+ schema: z+ .any()+ .optional()+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),modes: z.enum(["json", "git-diff"]).array().optional().default([]),tag: z.string().or(z.null()).default(null),})@@ -478,7 +585,11 @@ const extractOptions = z{message: "Invalid JSON schema.",},- ),+ )+ .transform(val => normalizeSchemaForOpenAI(val))+ .refine(val => validateSchemaForOpenAI(val), {+ message: OPENAI_SCHEMA_ERROR_MESSAGE,+ }),limit: z.number().int().positive().finite().safe().optional(),ignoreSitemap: z.boolean().default(false),includeSubdomains: z.boolean().default(true),diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex e6b4854da..47ce622d8 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -172,10 +172,15 @@ app.use(logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));}+ const customErrorMessage =+ err.errors.length > 0 && err.errors[0].code === "custom"+ ? err.errors[0].message+ : "Bad Request";+res.status(400).json({success: false,code: "BAD_REQUEST",- error: "Bad Request",+ error: customErrorMessage,details: err.errors,});} else {