Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixed spread schemas
Nicolasd547192f374b
diff --git a/apps/api/src/lib/__tests__/spread-schema-objects.test.ts b/apps/api/src/lib/__tests__/spread-schema-objects.test.tsindex 7cfcdd03d..f49fb378d 100644--- a/apps/api/src/lib/__tests__/spread-schema-objects.test.ts+++ b/apps/api/src/lib/__tests__/spread-schema-objects.test.ts@@ -283,4 +283,574 @@ describe("spreadSchemas", () => {expect(singleAnswerSchema).toEqual({});expect(multiEntitySchema).toEqual(schema);});++ it("should spread pages schema", async () => {+ const schema = {+ type: "object",+ properties: {+ pages: {+ type: "array",+ items: {+ type: "object",+ properties: {+ title: {+ type: "string",+ },+ },+ },+ },+ },+ required: ["pages"],+ };++ const keys = ["pages"];+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ schema,+ keys,+ );++ expect(singleAnswerSchema).toEqual({});+ expect(multiEntitySchema).toEqual(schema);+ });++ it("should spread pages schema", async () => {+ const schema = {+ type: "object",+ properties: {+ pages: {+ type: "array",+ items: {+ type: "object",+ properties: {+ title: {+ type: "string",+ },+ },+ },+ },+ },+ required: ["pages"],+ };++ const keys = ["pages.title"];+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ schema,+ keys,+ );++ expect(singleAnswerSchema).toEqual({});+ expect(multiEntitySchema).toEqual(schema);+ });++ it("should handle deeply nested array properties", async () => {+ const schema = {+ type: "object",+ properties: {+ company: {+ type: "object",+ properties: {+ name: { type: "string" },+ departments: {+ type: "array",+ items: {+ type: "object",+ properties: {+ name: { type: "string" },+ employees: {+ type: "array",+ items: {+ type: "object",+ properties: {+ name: { type: "string" },+ role: { type: "string" },+ },+ },+ },+ },+ },+ },+ },+ },+ },+ required: ["company"],+ };++ const keys = ["company.departments.employees"];+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ schema,+ keys,+ );++ expect(singleAnswerSchema).toEqual({});+ expect(multiEntitySchema).toEqual(schema);+ });++ it("should handle multiple nested paths", async () => {+ const schema = {+ type: "object",+ properties: {+ user: {+ type: "object",+ properties: {+ name: { type: "string" },+ contacts: {+ type: "array",+ items: {+ type: "object",+ properties: {+ email: { type: "string" },+ phone: { type: "string" },+ },+ },+ },+ },+ },+ orders: {+ type: "array",+ items: {+ type: "object",+ properties: {+ id: { type: "string" },+ items: {+ type: "array",+ items: {+ type: "object",+ properties: {+ name: { type: "string" },+ quantity: { type: "number" },+ },+ },+ },+ },+ },+ },+ },+ required: ["user", "orders"],+ };++ const keys = ["user.contacts", "orders.items"];+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ schema,+ keys,+ );++ expect(singleAnswerSchema).toEqual({});+ expect(multiEntitySchema).toEqual(schema);+ });++ it("should handle mixed single and array properties", async () => {+ const schema = {+ type: "object",+ properties: {+ metadata: {+ type: "object",+ properties: {+ title: { type: "string" },+ description: { type: "string" },+ },+ },+ sections: {+ type: "array",+ items: {+ type: "object",+ properties: {+ title: { type: "string" },+ content: { type: "string" },+ },+ },+ },+ },+ required: ["metadata", "sections"],+ };++ const keys = ["sections"];+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ schema,+ keys,+ );++ expect(singleAnswerSchema).toEqual({+ type: "object",+ properties: {+ metadata: {+ type: "object",+ properties: {+ title: { type: "string" },+ description: { type: "string" },+ },+ },+ },+ required: ["metadata"],+ });++ expect(multiEntitySchema).toEqual({+ type: "object",+ properties: {+ sections: {+ type: "array",+ items: {+ type: "object",+ properties: {+ title: { type: "string" },+ content: { type: "string" },+ },+ },+ },+ },+ required: ["sections"],+ });+ });++ it("should handle empty keys array", async () => {+ const schema = {+ type: "object",+ properties: {+ name: { type: "string" },+ age: { type: "number" },+ },+ required: ["name"],+ };++ const keys: string[] = [];+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ schema,+ keys,+ );++ expect(singleAnswerSchema).toEqual(schema);+ expect(multiEntitySchema).toEqual({});+ });++ it("should handle non-existent paths", async () => {+ const schema = {+ type: "object",+ properties: {+ user: {+ type: "object",+ properties: {+ name: { type: "string" },+ },+ },+ },+ };++ const keys = ["user.nonexistent.path"];+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ schema,+ keys,+ );++ expect(singleAnswerSchema).toEqual({});+ expect(multiEntitySchema).toEqual(schema);+ });++ // it("should split nested object and array properties", async () => {+ // const schema = {+ // type: "object",+ // properties: {+ // company: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // address: {+ // type: "object",+ // properties: {+ // street: { type: "string" },+ // city: { type: "string" },+ // },+ // },+ // employees: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // position: { type: "string" },+ // },+ // },+ // },+ // },+ // },+ // },+ // required: ["company"],+ // };++ // const keys = ["company.employees"];+ // const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ // schema,+ // keys,+ // );++ // expect(singleAnswerSchema).toEqual({+ // type: "object",+ // properties: {+ // company: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // address: {+ // type: "object",+ // properties: {+ // street: { type: "string" },+ // city: { type: "string" },+ // },+ // },+ // },+ // },+ // },+ // required: ["company"],+ // });++ // expect(multiEntitySchema).toEqual({+ // type: "object",+ // properties: {+ // company: {+ // type: "object",+ // properties: {+ // employees: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // position: { type: "string" },+ // },+ // },+ // },+ // },+ // },+ // },+ // required: ["company"],+ // });+ // });++ // it("should handle multiple root level properties with nested paths", async () => {+ // const schema = {+ // type: "object",+ // properties: {+ // user: {+ // type: "object",+ // properties: {+ // id: { type: "string" },+ // profile: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // email: { type: "string" },+ // },+ // },+ // posts: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // title: { type: "string" },+ // content: { type: "string" },+ // },+ // },+ // },+ // },+ // },+ // settings: {+ // type: "object",+ // properties: {+ // theme: { type: "string" },+ // notifications: {+ // type: "object",+ // properties: {+ // email: { type: "boolean" },+ // push: { type: "boolean" },+ // },+ // },+ // },+ // },+ // },+ // required: ["user", "settings"],+ // };++ // const keys = ["user.posts", "settings.notifications"];+ // const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ // schema,+ // keys,+ // );++ // expect(singleAnswerSchema).toEqual({+ // type: "object",+ // properties: {+ // user: {+ // type: "object",+ // properties: {+ // id: { type: "string" },+ // profile: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // email: { type: "string" },+ // },+ // },+ // },+ // },+ // settings: {+ // type: "object",+ // properties: {+ // theme: { type: "string" },+ // },+ // },+ // },+ // required: ["user", "settings"],+ // });++ // expect(multiEntitySchema).toEqual({+ // type: "object",+ // properties: {+ // user: {+ // type: "object",+ // properties: {+ // posts: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // title: { type: "string" },+ // content: { type: "string" },+ // },+ // },+ // },+ // },+ // },+ // settings: {+ // type: "object",+ // properties: {+ // notifications: {+ // type: "object",+ // properties: {+ // email: { type: "boolean" },+ // push: { type: "boolean" },+ // },+ // },+ // },+ // },+ // },+ // required: ["user", "settings"],+ // });+ // });++ // it("should handle array properties at different nesting levels", async () => {+ // const schema = {+ // type: "object",+ // properties: {+ // categories: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // subcategories: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // products: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // price: { type: "number" },+ // },+ // },+ // },+ // },+ // },+ // },+ // },+ // },+ // },+ // featured: {+ // type: "object",+ // properties: {+ // category: { type: "string" },+ // items: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // id: { type: "string" },+ // name: { type: "string" },+ // },+ // },+ // },+ // },+ // },+ // },+ // };++ // const keys = ["categories.subcategories", "featured.items"];+ // const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(+ // schema,+ // keys,+ // );++ // expect(singleAnswerSchema).toEqual({+ // type: "object",+ // properties: {+ // featured: {+ // type: "object",+ // properties: {+ // category: { type: "string" },+ // },+ // },+ // },+ // });++ // expect(multiEntitySchema).toEqual({+ // type: "object",+ // properties: {+ // categories: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // subcategories: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // products: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // name: { type: "string" },+ // price: { type: "number" },+ // },+ // },+ // },+ // },+ // },+ // },+ // },+ // },+ // },+ // featured: {+ // type: "object",+ // properties: {+ // items: {+ // type: "array",+ // items: {+ // type: "object",+ // properties: {+ // id: { type: "string" },+ // name: { type: "string" },+ // },+ // },+ // },+ // },+ // },+ // },+ // });+ // });});diff --git a/apps/api/src/lib/extract/helpers/spread-schemas.ts b/apps/api/src/lib/extract/helpers/spread-schemas.tsindex 8ba9f4288..c4026238b 100644--- a/apps/api/src/lib/extract/helpers/spread-schemas.ts+++ b/apps/api/src/lib/extract/helpers/spread-schemas.ts@@ -1,3 +1,5 @@+import { logger } from "../../../lib/logger";+export async function spreadSchemas(schema: any,keys: string[],@@ -6,14 +8,45 @@ export async function spreadSchemas(multiEntitySchema: any;}> {let singleAnswerSchema = { ...schema, properties: { ...schema.properties } };- let multiEntitySchema: any = { type: "object", properties: {} };+ let multiEntitySchema: any = {+ type: "object",+ properties: {},+ ...(schema.required ? { required: [] } : {})+ };++ // Helper function to check if a property path exists in schema+ const hasPropertyPath = (schema: any, path: string[]): boolean => {+ let current = schema.properties;+ for (let i = 0; i < path.length; i++) {+ if (!current[path[i]]) return false;+ if (current[path[i]].type === "array" && current[path[i]].items) {+ current = current[path[i]].items.properties;+ } else {+ current = current[path[i]].properties;+ }+ }+ return true;+ };++ // Helper function to get the root property of a dot path+ const getRootProperty = (path: string): string => {+ return path.split('.')[0];+ };keys.forEach((key) => {- if (singleAnswerSchema.properties[key]) {- multiEntitySchema.properties[key] = singleAnswerSchema.properties[key];- delete singleAnswerSchema.properties[key];+ const rootProperty = getRootProperty(key);+ if (singleAnswerSchema.properties[rootProperty]) {+ multiEntitySchema.properties[rootProperty] = singleAnswerSchema.properties[rootProperty];+ delete singleAnswerSchema.properties[rootProperty];++ // Move required field if it exists+ if (schema.required?.includes(rootProperty)) {+ multiEntitySchema.required.push(rootProperty);+ singleAnswerSchema.required = schema.required.filter((k: string) => k !== rootProperty);+ }}});+// Recursively delete empty properties in singleAnswerSchemaconst deleteEmptyProperties = (schema: any) => {for (const key in schema.properties) {@@ -34,10 +67,14 @@ export async function spreadSchemas(// If singleAnswerSchema has no properties left, return an empty objectif (Object.keys(singleAnswerSchema.properties).length === 0) {singleAnswerSchema = {};+ } else if (singleAnswerSchema.required?.length === 0) {+ delete singleAnswerSchema.required;}if (Object.keys(multiEntitySchema.properties).length === 0) {multiEntitySchema = {};+ } else if (multiEntitySchema.required?.length === 0) {+ delete multiEntitySchema.required;}return {