Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: fixed spread schemas

Nicolasd547192f374b
diff --git a/apps/api/src/lib/__tests__/spread-schema-objects.test.ts b/apps/api/src/lib/__tests__/spread-schema-objects.test.ts
index 7cfcdd03d..f49fb378d 100644
--- a/apps/api/src/lib/__tests__/spread-schema-objects.test.ts
+++ b/apps/api/src/lib/__tests__/spread-schema-objects.test.ts
@@ -283,4 +283,574 @@ describe("spreadSchemas", () => {
expect(singleAnswerSchema).toEqual({});
expect(multiEntitySchema).toEqual(schema);
});
+
+ it("should spread pages schema", async () => {
+ const schema = {
+ type: "object",
+ properties: {
+ pages: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ title: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ required: ["pages"],
+ };
+
+ const keys = ["pages"];
+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(
+ schema,
+ keys,
+ );
+
+ expect(singleAnswerSchema).toEqual({});
+ expect(multiEntitySchema).toEqual(schema);
+ });
+
+ it("should spread pages schema", async () => {
+ const schema = {
+ type: "object",
+ properties: {
+ pages: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ title: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ required: ["pages"],
+ };
+
+ const keys = ["pages.title"];
+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(
+ schema,
+ keys,
+ );
+
+ expect(singleAnswerSchema).toEqual({});
+ expect(multiEntitySchema).toEqual(schema);
+ });
+
+ it("should handle deeply nested array properties", async () => {
+ const schema = {
+ type: "object",
+ properties: {
+ company: {
+ type: "object",
+ properties: {
+ name: { type: "string" },
+ departments: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ name: { type: "string" },
+ employees: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ name: { type: "string" },
+ role: { type: "string" },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ required: ["company"],
+ };
+
+ const keys = ["company.departments.employees"];
+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(
+ schema,
+ keys,
+ );
+
+ expect(singleAnswerSchema).toEqual({});
+ expect(multiEntitySchema).toEqual(schema);
+ });
+
+ it("should handle multiple nested paths", async () => {
+ const schema = {
+ type: "object",
+ properties: {
+ user: {
+ type: "object",
+ properties: {
+ name: { type: "string" },
+ contacts: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ email: { type: "string" },
+ phone: { type: "string" },
+ },
+ },
+ },
+ },
+ },
+ orders: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ id: { type: "string" },
+ items: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ name: { type: "string" },
+ quantity: { type: "number" },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ required: ["user", "orders"],
+ };
+
+ const keys = ["user.contacts", "orders.items"];
+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(
+ schema,
+ keys,
+ );
+
+ expect(singleAnswerSchema).toEqual({});
+ expect(multiEntitySchema).toEqual(schema);
+ });
+
+ it("should handle mixed single and array properties", async () => {
+ const schema = {
+ type: "object",
+ properties: {
+ metadata: {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ description: { type: "string" },
+ },
+ },
+ sections: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ content: { type: "string" },
+ },
+ },
+ },
+ },
+ required: ["metadata", "sections"],
+ };
+
+ const keys = ["sections"];
+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(
+ schema,
+ keys,
+ );
+
+ expect(singleAnswerSchema).toEqual({
+ type: "object",
+ properties: {
+ metadata: {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ description: { type: "string" },
+ },
+ },
+ },
+ required: ["metadata"],
+ });
+
+ expect(multiEntitySchema).toEqual({
+ type: "object",
+ properties: {
+ sections: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ title: { type: "string" },
+ content: { type: "string" },
+ },
+ },
+ },
+ },
+ required: ["sections"],
+ });
+ });
+
+ it("should handle empty keys array", async () => {
+ const schema = {
+ type: "object",
+ properties: {
+ name: { type: "string" },
+ age: { type: "number" },
+ },
+ required: ["name"],
+ };
+
+ const keys: string[] = [];
+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(
+ schema,
+ keys,
+ );
+
+ expect(singleAnswerSchema).toEqual(schema);
+ expect(multiEntitySchema).toEqual({});
+ });
+
+ it("should handle non-existent paths", async () => {
+ const schema = {
+ type: "object",
+ properties: {
+ user: {
+ type: "object",
+ properties: {
+ name: { type: "string" },
+ },
+ },
+ },
+ };
+
+ const keys = ["user.nonexistent.path"];
+ const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(
+ schema,
+ keys,
+ );
+
+ expect(singleAnswerSchema).toEqual({});
+ expect(multiEntitySchema).toEqual(schema);
+ });
+
+ // it("should split nested object and array properties", async () => {
+ // const schema = {
+ // type: "object",
+ // properties: {
+ // company: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // address: {
+ // type: "object",
+ // properties: {
+ // street: { type: "string" },
+ // city: { type: "string" },
+ // },
+ // },
+ // employees: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // position: { type: "string" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // required: ["company"],
+ // };
+
+ // const keys = ["company.employees"];
+ // const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(
+ // schema,
+ // keys,
+ // );
+
+ // expect(singleAnswerSchema).toEqual({
+ // type: "object",
+ // properties: {
+ // company: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // address: {
+ // type: "object",
+ // properties: {
+ // street: { type: "string" },
+ // city: { type: "string" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // required: ["company"],
+ // });
+
+ // expect(multiEntitySchema).toEqual({
+ // type: "object",
+ // properties: {
+ // company: {
+ // type: "object",
+ // properties: {
+ // employees: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // position: { type: "string" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // required: ["company"],
+ // });
+ // });
+
+ // it("should handle multiple root level properties with nested paths", async () => {
+ // const schema = {
+ // type: "object",
+ // properties: {
+ // user: {
+ // type: "object",
+ // properties: {
+ // id: { type: "string" },
+ // profile: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // email: { type: "string" },
+ // },
+ // },
+ // posts: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // title: { type: "string" },
+ // content: { type: "string" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // settings: {
+ // type: "object",
+ // properties: {
+ // theme: { type: "string" },
+ // notifications: {
+ // type: "object",
+ // properties: {
+ // email: { type: "boolean" },
+ // push: { type: "boolean" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // required: ["user", "settings"],
+ // };
+
+ // const keys = ["user.posts", "settings.notifications"];
+ // const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(
+ // schema,
+ // keys,
+ // );
+
+ // expect(singleAnswerSchema).toEqual({
+ // type: "object",
+ // properties: {
+ // user: {
+ // type: "object",
+ // properties: {
+ // id: { type: "string" },
+ // profile: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // email: { type: "string" },
+ // },
+ // },
+ // },
+ // },
+ // settings: {
+ // type: "object",
+ // properties: {
+ // theme: { type: "string" },
+ // },
+ // },
+ // },
+ // required: ["user", "settings"],
+ // });
+
+ // expect(multiEntitySchema).toEqual({
+ // type: "object",
+ // properties: {
+ // user: {
+ // type: "object",
+ // properties: {
+ // posts: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // title: { type: "string" },
+ // content: { type: "string" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // settings: {
+ // type: "object",
+ // properties: {
+ // notifications: {
+ // type: "object",
+ // properties: {
+ // email: { type: "boolean" },
+ // push: { type: "boolean" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // required: ["user", "settings"],
+ // });
+ // });
+
+ // it("should handle array properties at different nesting levels", async () => {
+ // const schema = {
+ // type: "object",
+ // properties: {
+ // categories: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // subcategories: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // products: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // price: { type: "number" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // featured: {
+ // type: "object",
+ // properties: {
+ // category: { type: "string" },
+ // items: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // id: { type: "string" },
+ // name: { type: "string" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // };
+
+ // const keys = ["categories.subcategories", "featured.items"];
+ // const { singleAnswerSchema, multiEntitySchema } = await spreadSchemas(
+ // schema,
+ // keys,
+ // );
+
+ // expect(singleAnswerSchema).toEqual({
+ // type: "object",
+ // properties: {
+ // featured: {
+ // type: "object",
+ // properties: {
+ // category: { type: "string" },
+ // },
+ // },
+ // },
+ // });
+
+ // expect(multiEntitySchema).toEqual({
+ // type: "object",
+ // properties: {
+ // categories: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // subcategories: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // products: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // name: { type: "string" },
+ // price: { type: "number" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // featured: {
+ // type: "object",
+ // properties: {
+ // items: {
+ // type: "array",
+ // items: {
+ // type: "object",
+ // properties: {
+ // id: { type: "string" },
+ // name: { type: "string" },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // },
+ // });
+ // });
});
diff --git a/apps/api/src/lib/extract/helpers/spread-schemas.ts b/apps/api/src/lib/extract/helpers/spread-schemas.ts
index 8ba9f4288..c4026238b 100644
--- a/apps/api/src/lib/extract/helpers/spread-schemas.ts
+++ b/apps/api/src/lib/extract/helpers/spread-schemas.ts
@@ -1,3 +1,5 @@
+import { logger } from "../../../lib/logger";
+
export async function spreadSchemas(
schema: any,
keys: string[],
@@ -6,14 +8,45 @@ export async function spreadSchemas(
multiEntitySchema: any;
}> {
let singleAnswerSchema = { ...schema, properties: { ...schema.properties } };
- let multiEntitySchema: any = { type: "object", properties: {} };
+ let multiEntitySchema: any = {
+ type: "object",
+ properties: {},
+ ...(schema.required ? { required: [] } : {})
+ };
+
+ // Helper function to check if a property path exists in schema
+ const hasPropertyPath = (schema: any, path: string[]): boolean => {
+ let current = schema.properties;
+ for (let i = 0; i < path.length; i++) {
+ if (!current[path[i]]) return false;
+ if (current[path[i]].type === "array" && current[path[i]].items) {
+ current = current[path[i]].items.properties;
+ } else {
+ current = current[path[i]].properties;
+ }
+ }
+ return true;
+ };
+
+ // Helper function to get the root property of a dot path
+ const getRootProperty = (path: string): string => {
+ return path.split('.')[0];
+ };
keys.forEach((key) => {
- if (singleAnswerSchema.properties[key]) {
- multiEntitySchema.properties[key] = singleAnswerSchema.properties[key];
- delete singleAnswerSchema.properties[key];
+ const rootProperty = getRootProperty(key);
+ if (singleAnswerSchema.properties[rootProperty]) {
+ multiEntitySchema.properties[rootProperty] = singleAnswerSchema.properties[rootProperty];
+ delete singleAnswerSchema.properties[rootProperty];
+
+ // Move required field if it exists
+ if (schema.required?.includes(rootProperty)) {
+ multiEntitySchema.required.push(rootProperty);
+ singleAnswerSchema.required = schema.required.filter((k: string) => k !== rootProperty);
+ }
}
});
+
// Recursively delete empty properties in singleAnswerSchema
const deleteEmptyProperties = (schema: any) => {
for (const key in schema.properties) {
@@ -34,10 +67,14 @@ export async function spreadSchemas(
// If singleAnswerSchema has no properties left, return an empty object
if (Object.keys(singleAnswerSchema.properties).length === 0) {
singleAnswerSchema = {};
+ } else if (singleAnswerSchema.required?.length === 0) {
+ delete singleAnswerSchema.required;
}
if (Object.keys(multiEntitySchema.properties).length === 0) {
multiEntitySchema = {};
+ } else if (multiEntitySchema.required?.length === 0) {
+ delete multiEntitySchema.required;
}
return {