Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: enum arrays being converted to objects (#2224)

Gaurav Chadhae8b5f562a0c4
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.ts
index d6f95edf7..5034bcc3f 100644
--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts
@@ -7,6 +7,7 @@ import {
idmux,
Identity,
scrapeRaw,
+ extract,
} from "./lib";
import crypto from "crypto";
@@ -1574,4 +1575,206 @@ describe("Scrape tests", () => {
scrapeTimeout,
);
});
+
+ describe("Schema validation for additionalProperties", () => {
+ if (
+ !process.env.TEST_SUITE_SELF_HOSTED ||
+ process.env.OPENAI_API_KEY ||
+ process.env.OLLAMA_BASE_URL
+ ) {
+ it(
+ "should normalize scrape request with additionalProperties in extract schema",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: ["extract"],
+ extract: {
+ schema: {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ },
+ additionalProperties: false,
+ },
+ },
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should normalize extract request with additionalProperties in schema",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await extract(
+ {
+ urls: ["https://example.com"],
+ schema: {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ },
+ additionalProperties: true,
+ },
+ },
+ identity,
+ );
+
+ expect(response.success).toBe(true);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should normalize scrape request with nested additionalProperties",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: ["extract"],
+ extract: {
+ schema: {
+ type: "object",
+ properties: {
+ user: {
+ type: "object",
+ properties: {
+ name: { type: "string" },
+ },
+ additionalProperties: false,
+ },
+ },
+ },
+ },
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should accept valid schema without additionalProperties",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: ["extract"],
+ extract: {
+ schema: {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ },
+ required: ["title"],
+ },
+ },
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should normalize changeTracking with additionalProperties in schema",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: ["markdown", "changeTracking"],
+ changeTrackingOptions: {
+ schema: {
+ type: "object",
+ properties: {
+ changes: { type: "string" },
+ },
+ additionalProperties: true,
+ },
+ },
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should reject schema-less dictionary (no properties but additionalProperties: true)",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: ["extract"],
+ extract: {
+ schema: {
+ type: "object",
+ additionalProperties: true,
+ },
+ },
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(400);
+ expect(response.body.error).toContain("OpenAI");
+ expect(response.body.error).toContain("schema-less dictionary");
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should normalize scrape request with object type without properties (but no additionalProperties)",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: ["extract"],
+ extract: {
+ schema: {
+ type: "object",
+ properties: {
+ address: { type: "string" },
+ detail: {
+ type: "object",
+ description:
+ "Any other specifications of the particular make and model in the page",
+ },
+ },
+ },
+ },
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+ }
+ });
});
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts
index 6ab7c96f1..47c4e1750 100644
--- a/apps/api/src/__tests__/snips/v2/lib.ts
+++ b/apps/api/src/__tests__/snips/v2/lib.ts
@@ -472,6 +472,69 @@ export async function batchScrapeWithConcurrencyTracking(
};
}
+// =========================================
+// Extract API
+// =========================================
+
+async function extractStart(body: ExtractRequestInput, identity: Identity) {
+ return await request(TEST_URL)
+ .post("/v2/extract")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send(body);
+}
+
+async function extractStatus(id: string, identity: Identity) {
+ return await request(TEST_URL)
+ .get("/v2/extract/" + encodeURIComponent(id))
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .send();
+}
+
+function expectExtractStartToSucceed(
+ response: Awaited<ReturnType<typeof extractStart>>,
+) {
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(typeof response.body.id).toBe("string");
+}
+
+function expectExtractToSucceed(
+ response: Awaited<ReturnType<typeof extractStatus>>,
+) {
+ expect(response.statusCode).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(typeof response.body.status).toBe("string");
+ expect(response.body.status).toBe("completed");
+ expect(response.body).toHaveProperty("data");
+}
+
+export async function extract(
+ body: ExtractRequestInput,
+ identity: Identity,
+): Promise<ExtractResponse> {
+ const es = await extractStart(body, identity);
+ expectExtractStartToSucceed(es);
+
+ let x;
+
+ do {
+ x = await extractStatus(es.body.id, identity);
+ expect(x.statusCode).toBe(200);
+ expect(typeof x.body.status).toBe("string");
+ } while (x.body.status === "processing");
+
+ expectExtractToSucceed(x);
+ return x.body;
+}
+
+export async function extractRaw(
+ body: ExtractRequestInput,
+ identity: Identity,
+) {
+ return await extractStart(body, identity);
+}
+
// =========================================
// ZDR API
// =========================================
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index 2f6f57cc6..a1d20c658 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -7,6 +7,7 @@ import {
idmux,
Identity,
scrapeRaw,
+ extractRaw,
} from "./lib";
import crypto from "crypto";
@@ -1714,4 +1715,185 @@ describe("attributes format", () => {
},
scrapeTimeout,
);
+
+ describe("Schema validation for additionalProperties", () => {
+ if (
+ !process.env.TEST_SUITE_SELF_HOSTED ||
+ process.env.OPENAI_API_KEY ||
+ process.env.OLLAMA_BASE_URL
+ ) {
+ it(
+ "should normalize scrape request with additionalProperties in json format schema",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ },
+ additionalProperties: false,
+ },
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should normalize extract request with additionalProperties in schema",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await extractRaw(
+ {
+ urls: ["https://example.com"],
+ schema: {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ },
+ additionalProperties: true,
+ },
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should normalize changeTracking format with additionalProperties",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: [
+ { type: "markdown" },
+ {
+ type: "changeTracking",
+ schema: {
+ type: "object",
+ properties: {
+ changes: { type: "string" },
+ },
+ additionalProperties: false,
+ },
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should accept valid schema without additionalProperties",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ },
+ required: ["title"],
+ },
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should reject schema-less dictionary (no properties but additionalProperties: true)",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ additionalProperties: true,
+ },
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(400);
+ expect(response.body.error).toContain("OpenAI");
+ expect(response.body.error).toContain("schema-less dictionary");
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "should normalize schema with object type without properties (but no additionalProperties)",
+ async () => {
+ const identity = await idmux({ name: "schema-validation-test" });
+
+ const response = await scrapeRaw(
+ {
+ url: "https://example.com",
+ formats: [
+ {
+ type: "json",
+ schema: {
+ type: "object",
+ properties: {
+ address: { type: "string" },
+ detail: {
+ type: "object",
+ description:
+ "Any other specifications of the particular make and model in the page",
+ },
+ },
+ },
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(200);
+ },
+ scrapeTimeout,
+ );
+ }
+ });
});
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index b9f0419ef..1f7d8a4da 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -74,6 +74,100 @@ const agentExtractModelValue = "fire-1";
export const isAgentExtractModelValid = (x: string | undefined) =>
x?.toLowerCase() === agentExtractModelValue;
+function normalizeSchemaForOpenAI(schema: any): any {
+ if (!schema || typeof schema !== "object") {
+ return schema;
+ }
+
+ const visited = new WeakSet();
+
+ function normalizeObject(obj: any): any {
+ if (typeof obj !== "object" || obj === null) return obj;
+ if (Array.isArray(obj)) return obj;
+
+ if (visited.has(obj)) return obj;
+ visited.add(obj);
+
+ const normalized = { ...obj };
+
+ if (
+ normalized.type === "object" &&
+ normalized.hasOwnProperty("properties") &&
+ normalized.hasOwnProperty("additionalProperties")
+ ) {
+ delete normalized.additionalProperties;
+ }
+
+ if (
+ normalized.type === "object" &&
+ normalized.hasOwnProperty("required") &&
+ normalized.hasOwnProperty("properties")
+ ) {
+ if (
+ Array.isArray(normalized.required) &&
+ typeof normalized.properties === "object" &&
+ normalized.properties !== null
+ ) {
+ const validRequired = normalized.required.filter((field: string) =>
+ normalized.properties.hasOwnProperty(field),
+ );
+ if (validRequired.length > 0) {
+ normalized.required = validRequired;
+ } else {
+ delete normalized.required;
+ }
+ } else {
+ delete normalized.required;
+ }
+ }
+
+ for (const [key, value] of Object.entries(normalized)) {
+ if (typeof value === "object" && value !== null) {
+ normalized[key] = normalizeObject(value);
+ }
+ }
+
+ return normalized;
+ }
+
+ return normalizeObject(schema);
+}
+function validateSchemaForOpenAI(schema: any): boolean {
+ if (!schema || typeof schema !== "object") {
+ return true;
+ }
+
+ const visited = new WeakSet();
+
+ function hasInvalidStructure(obj: any): boolean {
+ if (typeof obj !== "object" || obj === null) return false;
+
+ if (visited.has(obj)) return false;
+ visited.add(obj);
+
+ if (
+ obj.type === "object" &&
+ !obj.hasOwnProperty("properties") &&
+ !obj.hasOwnProperty("patternProperties") &&
+ obj.additionalProperties === true
+ ) {
+ return true;
+ }
+
+ for (const value of Object.values(obj)) {
+ if (typeof value === "object" && value !== null) {
+ if (hasInvalidStructure(value)) return true;
+ }
+ }
+ return false;
+ }
+
+ return !hasInvalidStructure(schema);
+}
+
+const OPENAI_SCHEMA_ERROR_MESSAGE =
+ "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";
+
export const agentOptionsExtract = z
.object({
model: z.string().default(agentExtractModelValue),
@@ -99,7 +193,11 @@ export const extractOptions = z
{
message: "Invalid JSON schema.",
},
- ),
+ )
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
systemPrompt: z.string().max(10000).default(""),
prompt: z.string().max(10000).optional(),
temperature: z.number().optional(),
@@ -130,7 +228,11 @@ const extractOptionsWithAgent = z
{
message: "Invalid JSON schema.",
},
- ),
+ )
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
systemPrompt: z.string().max(10000).default(""),
prompt: z.string().max(10000).optional(),
temperature: z.number().optional(),
@@ -367,7 +469,11 @@ const baseScrapeOptions = z
{
message: "Invalid JSON schema.",
},
- ),
+ )
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
modes: z.enum(["json", "git-diff"]).array().optional().default([]),
tag: z.string().or(z.null()).default(null),
})
@@ -573,7 +679,11 @@ const extractV1Options = z
{
message: "Invalid JSON schema.",
},
- ),
+ )
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
limit: z.number().int().positive().finite().safe().optional(),
ignoreSitemap: z.boolean().default(false),
includeSubdomains: z.boolean().default(true),
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 167299396..dd7c0c168 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -63,6 +63,101 @@ export const url = z.preprocess(
const strictMessage =
"Unrecognized key in body -- please review the v2 API documentation for request body changes";
+function normalizeSchemaForOpenAI(schema: any): any {
+ if (!schema || typeof schema !== "object") {
+ return schema;
+ }
+
+ const visited = new WeakSet();
+
+ function normalizeObject(obj: any): any {
+ if (typeof obj !== "object" || obj === null) return obj;
+ if (Array.isArray(obj)) return obj;
+
+ if (visited.has(obj)) return obj;
+ visited.add(obj);
+
+ const normalized = { ...obj };
+
+ if (
+ normalized.type === "object" &&
+ normalized.hasOwnProperty("properties") &&
+ normalized.hasOwnProperty("additionalProperties")
+ ) {
+ delete normalized.additionalProperties;
+ }
+
+ if (
+ normalized.type === "object" &&
+ normalized.hasOwnProperty("required") &&
+ normalized.hasOwnProperty("properties")
+ ) {
+ if (
+ Array.isArray(normalized.required) &&
+ typeof normalized.properties === "object" &&
+ normalized.properties !== null
+ ) {
+ const validRequired = normalized.required.filter((field: string) =>
+ normalized.properties.hasOwnProperty(field),
+ );
+ if (validRequired.length > 0) {
+ normalized.required = validRequired;
+ } else {
+ delete normalized.required;
+ }
+ } else {
+ delete normalized.required;
+ }
+ }
+
+ for (const [key, value] of Object.entries(normalized)) {
+ if (typeof value === "object" && value !== null) {
+ normalized[key] = normalizeObject(value);
+ }
+ }
+
+ return normalized;
+ }
+
+ return normalizeObject(schema);
+}
+
+function validateSchemaForOpenAI(schema: any): boolean {
+ if (!schema || typeof schema !== "object") {
+ return true;
+ }
+
+ const visited = new WeakSet();
+
+ function hasInvalidStructure(obj: any): boolean {
+ if (typeof obj !== "object" || obj === null) return false;
+
+ if (visited.has(obj)) return false;
+ visited.add(obj);
+
+ if (
+ obj.type === "object" &&
+ !obj.hasOwnProperty("properties") &&
+ !obj.hasOwnProperty("patternProperties") &&
+ obj.additionalProperties === true
+ ) {
+ return true;
+ }
+
+ for (const value of Object.values(obj)) {
+ if (typeof value === "object" && value !== null) {
+ if (hasInvalidStructure(value)) return true;
+ }
+ }
+ return false;
+ }
+
+ return !hasInvalidStructure(schema);
+}
+
+const OPENAI_SCHEMA_ERROR_MESSAGE =
+ "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object.";
+
const ACTIONS_MAX_WAIT_TIME = 60;
const MAX_ACTIONS = 50;
function calculateTotalWaitTime(
@@ -174,7 +269,13 @@ const actionsSchema = z
const jsonFormatWithOptions = z
.object({
type: z.literal("json"),
- schema: z.any().optional(),
+ schema: z
+ .any()
+ .optional()
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
prompt: z.string().max(10000).optional(),
})
.strict();
@@ -185,7 +286,13 @@ const changeTrackingFormatWithOptions = z
.object({
type: z.literal("changeTracking"),
prompt: z.string().optional(),
- schema: z.any().optional(),
+ schema: z
+ .any()
+ .optional()
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
modes: z.enum(["json", "git-diff"]).array().optional().default([]),
tag: z.string().or(z.null()).default(null),
})
@@ -478,7 +585,11 @@ const extractOptions = z
{
message: "Invalid JSON schema.",
},
- ),
+ )
+ .transform(val => normalizeSchemaForOpenAI(val))
+ .refine(val => validateSchemaForOpenAI(val), {
+ message: OPENAI_SCHEMA_ERROR_MESSAGE,
+ }),
limit: z.number().int().positive().finite().safe().optional(),
ignoreSitemap: z.boolean().default(false),
includeSubdomains: z.boolean().default(true),
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index e6b4854da..47ce622d8 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -172,10 +172,15 @@ app.use(
logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));
}
+ const customErrorMessage =
+ err.errors.length > 0 && err.errors[0].code === "custom"
+ ? err.errors[0].message
+ : "Bad Request";
+
res.status(400).json({
success: false,
code: "BAD_REQUEST",
- error: "Bad Request",
+ error: customErrorMessage,
details: err.errors,
});
} else {