Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(llmExtract): remove unsupported JSON schema properties (#1335)
Gergő Móriczc3ebfafba7fa
diff --git a/apps/api/src/__tests__/snips/extract.test.ts b/apps/api/src/__tests__/snips/extract.test.tsindex 50fecde3c..acba310d0 100644--- a/apps/api/src/__tests__/snips/extract.test.ts+++ b/apps/api/src/__tests__/snips/extract.test.ts@@ -29,6 +29,28 @@ describe("Extract tests", () => {expect(typeof res.data.is_open_source).toBe("boolean");expect(res.data.is_open_source).toBe(true);}, 60000);++ it.concurrent("works with unsupported JSON schema parameters", async () => {+ const res = await extract({+ urls: ["https://firecrawl.dev"],+ schema: {+ "type": "object",+ "properties": {+ "company_name": {+ "type": "string",+ "pattern": "^[a-zA-Z0-9]+$"+ },+ },+ "required": [+ "company_name"+ ]+ },+ origin: "api-sdk",+ });++ expect(res.data).toHaveProperty("company_name");+ expect(typeof res.data.company_name).toBe("string")+ }, 60000);} else {it.concurrent("dummy test", () => {expect(true).toBe(true);diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 256b24d9e..e3c76362e 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -365,7 +365,37 @@ export async function performLLMExtract(export function removeDefaultProperty(schema: any): any {if (typeof schema !== "object" || schema === null) return schema;- const { default: _, ...rest } = schema;+ const rest = { ...schema };++ // unsupported global keys+ delete rest.default;++ // unsupported object keys+ delete rest.patternProperties;+ delete rest.unevaluatedProperties;+ delete rest.propertyNames;+ delete rest.minProperties;+ delete rest.maxProperties;++ // unsupported string keys+ delete rest.minLength;+ delete rest.maxLength;+ delete rest.pattern;+ delete rest.format;++ // unsupported number keys+ delete rest.minimum;+ delete rest.maximum;+ delete rest.multipleOf;++ // unsupported array keys+ delete rest.unevaluatedItems;+ delete rest.contains;+ delete rest.minContains;+ delete rest.maxContains;+ delete rest.minItems;+ delete rest.maxItems;+ delete rest.uniqueItems;for (const key in rest) {if (Array.isArray(rest[key])) {