Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): accept standard JSON Schema formats in v1 json and extract schemas (#5274)
Erik Engervall8e228da198ad
diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.tsindex 89dedbb10..9ba403ecb 100644--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts@@ -191,6 +191,48 @@ describe("V1 Types Validation", () => {expect(result.timeout).toBe(60000); // Should be transformed from 30000});+ it("should accept a jsonOptions schema that uses standard string formats", () => {+ const input: ScrapeRequestInput = {+ url: "https://example.com",+ formats: ["json"],+ jsonOptions: {+ schema: {+ type: "object",+ properties: {+ emails: {+ type: "array",+ items: { type: "string", format: "email" },+ },+ homepage: { type: "string", format: "uri" },+ },+ required: ["emails"],+ },+ },+ };++ const result = scrapeRequestSchema.parse(input);+ expect(result.jsonOptions).toBeDefined();+ });++ it("should reject a jsonOptions schema with an unknown format", () => {+ const input: ScrapeRequestInput = {+ url: "https://example.com",+ formats: ["json"],+ jsonOptions: {+ schema: {+ type: "object",+ properties: {+ email: { type: "string", format: "not-a-real-format" },+ },+ },+ },+ };++ expect(() => scrapeRequestSchema.parse(input)).toThrow(+ "Invalid JSON schema",+ );+ });+it("should reject extract options without extract format", () => {const input: ScrapeRequestInput = {url: "https://example.com",diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex 5d2e67851..4fee04eaf 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -835,6 +835,21 @@ describe("V2 Types Validation", () => {expect(result.prompt).toBe("Extract the title");});+ it("should accept an extract schema that uses standard string formats", () => {+ const input: ExtractRequestInput = {+ urls: ["https://example.com"],+ schema: {+ type: "object",+ properties: {+ contact: { type: "string", format: "email" },+ },+ },+ };++ const result = extractRequestSchema.parse(input);+ expect(result.schema).toBeDefined();+ });+it("should reject extract request without urls or prompt", () => {const input = {schema: {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 3614ee7ef..e259a5268 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -18,6 +18,7 @@ import {import { InternalOptions } from "../../scraper/scrapeURL";import { getURLDepth } from "../../scraper/WebScraper/utils/maxDepthUtils";import Ajv from "ajv";+import addFormats from "ajv-formats";import { ErrorCodes } from "../../lib/error";import { integrationSchema } from "../../utils/integration";import { includesFormat } from "../../lib/format-utils";@@ -744,6 +745,7 @@ export type ScrapeOptions = BaseScrapeOptions & {};const ajv = new Ajv();+addFormats(ajv);const extractV1Options = z.strictObject({diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 56d6afdce..8f6403f39 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1050,6 +1050,7 @@ export type UploadedParseFile = {};const ajv = new Ajv();+addFormats(ajv);const agentAjv = new Ajv();addFormats(agentAjv);