There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
feat(infra): orm document mode support (#8390)
forehalodbcbe9ce1af5
diff --git a/packages/common/infra/src/orm/core/__tests__/doc.spec.ts b/packages/common/infra/src/orm/core/__tests__/doc.spec.tsnew file mode 100644index 0000000000..2a220b0db1--- /dev/null+++ b/packages/common/infra/src/orm/core/__tests__/doc.spec.ts@@ -0,0 +1,127 @@+import {+ beforeEach,+ describe,+ expect,+ test as vitest,+ type TestAPI,+} from 'vitest';++import {+ createORMClient,+ type DBSchemaBuilder,+ f,+ MemoryORMAdapter,+ t,+ Table,+} from '../';++const TEST_SCHEMA = {+ docProperties: t.document({+ docId: f.string().primaryKey(),+ }),+} satisfies DBSchemaBuilder;++const ORMClient = createORMClient(TEST_SCHEMA);++type Context = {+ client: InstanceType<typeof ORMClient>;+};++beforeEach<Context>(async t => {+ t.client = new ORMClient(new MemoryORMAdapter());+});++const test = vitest as TestAPI<Context>;++describe('ORM entity CRUD', () => {+ test('still have type check', t => {+ const { client } = t;++ expect(() =>+ // @ts-expect-error type test+ client.docProperties.create({+ // docId missed+ prop1: 'prop1:value',+ prop2: 'prop2:value',+ })+ ).toThrow();+ });++ test('should be able to create ORM client', t => {+ const { client } = t;++ expect(client.docProperties instanceof Table).toBe(true);+ });++ test('should be able to create entity', async t => {+ const { client } = t;++ const doc = client.docProperties.create({+ docId: '1',+ prop1: 'prop1:value',+ prop2: 'prop2:value',+ });++ expect(doc.docId).toBe('1');+ expect(doc.prop1).toBe('prop1:value');+ expect(doc.prop2).toBe('prop2:value');+ });++ test('should be able to read entity', async t => {+ const { client } = t;++ const doc = client.docProperties.create({+ docId: '1',+ prop1: 'prop1:value',+ prop2: 'prop2:value',+ });++ const doc2 = client.docProperties.get(doc.docId);++ expect(doc2).toStrictEqual(doc);+ });++ test('should be able to update entity', async t => {+ const { client } = t;++ const doc = client.docProperties.create({+ docId: '1',+ prop1: 'prop1:value',+ prop2: 'prop2:value',+ });++ client.docProperties.update(doc.docId, {+ prop1: 'prop1:value2',+ prop3: 'prop3:value',+ prop4: null,+ prop5: undefined,+ });++ const doc2 = client.docProperties.get(doc.docId);++ expect(doc2).toStrictEqual({+ docId: '1',+ prop1: 'prop1:value2',+ prop2: 'prop2:value',+ prop3: 'prop3:value',+ prop4: null,+ prop5: undefined,+ });+ });++ test('should be able to delete entity', async t => {+ const { client } = t;++ const doc = client.docProperties.create({+ docId: '1',+ prop1: 'prop1:value',+ prop2: 'prop2:value',+ });++ client.docProperties.delete(doc.docId);++ const doc2 = client.docProperties.get(doc.docId);++ expect(doc2).toBe(null);+ });+});diff --git a/packages/common/infra/src/orm/core/__tests__/yjs.spec.ts b/packages/common/infra/src/orm/core/__tests__/yjs.spec.tsindex 2a78ca3ebd..77cd42ab26 100644--- a/packages/common/infra/src/orm/core/__tests__/yjs.spec.ts+++ b/packages/common/infra/src/orm/core/__tests__/yjs.spec.ts@@ -1,5 +1,11 @@import { nanoid } from 'nanoid';-import { beforeEach, describe, expect, test as t, type TestAPI } from 'vitest';+import {+ beforeEach,+ describe,+ expect,+ test as vitest,+ type TestAPI,+} from 'vitest';import { Doc } from 'yjs';import {@@ -8,6 +14,7 @@ import {type DocProvider,type Entity,f,+ t,Table,YjsDBAdapter,} from '../';@@ -28,6 +35,9 @@ const TEST_SCHEMA = {name: f.string(),email: f.string().optional(),},+ userInfo: t.document({+ userId: f.number().primaryKey(),+ }),} satisfies DBSchemaBuilder;const docProvider: DocProvider = {@@ -45,7 +55,7 @@ beforeEach<Context>(async t => {t.client = new Client(new YjsDBAdapter(TEST_SCHEMA, docProvider));});-const test = t as TestAPI<Context>;+const test = vitest as TestAPI<Context>;describe('ORM entity CRUD', () => {test('should be able to create ORM client', t => {@@ -404,4 +414,71 @@ describe('ORM entity CRUD', () => {expect(found).toEqual([]);}});++ test('should be able to create document entity', t => {+ const { client } = t;++ const doc = client.userInfo.create({+ userId: 1,+ avatar: 'avatar.jpg',+ address: '123 Main St',+ });++ expect(doc.userId).toBe(1);+ expect(doc.avatar).toBe('avatar.jpg');+ expect(doc.address).toBe('123 Main St');+ });++ test('should be able to read document entity', t => {+ const { client } = t;++ const doc = client.userInfo.create({+ userId: 1,+ avatar: 'avatar.jpg',+ address: '123 Main St',+ });++ const doc2 = client.userInfo.get(1);++ expect(doc2).toStrictEqual(doc);+ });++ test('should be able to update document entity', t => {+ const { client } = t;++ const doc = client.userInfo.create({+ userId: 1,+ avatar: 'avatar.jpg',+ address: '123 Main St',+ });++ client.userInfo.update(doc.userId, {+ avatar: 'avatar2.jpg',+ city: 'New York',+ });++ const doc2 = client.userInfo.get(1);++ expect(doc2).toStrictEqual({+ userId: 1,+ avatar: 'avatar2.jpg',+ address: '123 Main St',+ city: 'New York',+ });+ });++ test('should be able to delete document entity', t => {+ const { client } = t;++ const doc = client.userInfo.create({+ userId: 1,+ avatar: 'avatar.jpg',+ address: '123 Main St',+ });++ client.userInfo.delete(doc.userId);++ const doc2 = client.userInfo.get(1);+ expect(doc2).toBe(null);+ });});diff --git a/packages/common/infra/src/orm/core/adapters/yjs/table.ts b/packages/common/infra/src/orm/core/adapters/yjs/table.tsindex 712d646c71..e01f3f4cb3 100644--- a/packages/common/infra/src/orm/core/adapters/yjs/table.ts+++ b/packages/common/infra/src/orm/core/adapters/yjs/table.ts@@ -208,7 +208,7 @@ export class YjsTableAdapter implements TableAdapter {if (select === 'key') {return this.keyof(record);} else if (select === '*') {- selectedFields = this.fields;+ return this.toObject(record);} else {selectedFields = select;}diff --git a/packages/common/infra/src/orm/core/schema.ts b/packages/common/infra/src/orm/core/schema.tsindex 83f39f67c0..af28ef6eca 100644--- a/packages/common/infra/src/orm/core/schema.ts+++ b/packages/common/infra/src/orm/core/schema.ts@@ -12,6 +12,10 @@ export type TableSchemaBuilder = Record<string,FieldSchemaBuilder<any, boolean>>;+export type DocumentTableSchemaBuilder = TableSchemaBuilder & {+ __document: FieldSchemaBuilder<boolean, true, false>;+};+export type DBSchemaBuilder = Record<string, TableSchemaBuilder>;export class FieldSchemaBuilder<@@ -53,3 +57,12 @@ export const f = {boolean: () => new FieldSchemaBuilder<boolean>('boolean'),json: <T = any>() => new FieldSchemaBuilder<T>('json'),} satisfies Record<FieldType, () => FieldSchemaBuilder<any>>;++export const t = {+ document: <T extends TableSchemaBuilder>(schema: T) => {+ return {+ ...schema,+ __document: new FieldSchemaBuilder<boolean>('boolean').optional(),+ };+ },+};diff --git a/packages/common/infra/src/orm/core/table.ts b/packages/common/infra/src/orm/core/table.tsindex 8ac7c14da3..bad752d2d3 100644--- a/packages/common/infra/src/orm/core/table.ts+++ b/packages/common/infra/src/orm/core/table.ts@@ -4,6 +4,7 @@ import { Observable, shareReplay } from 'rxjs';import type { DBAdapter, TableAdapter } from './adapters';import type {DBSchemaBuilder,+ DocumentTableSchemaBuilder,FieldSchemaBuilder,TableSchema,TableSchemaBuilder,@@ -17,72 +18,115 @@ type Pretty<T> = T extends any}: never;+// filter out all fields starting with `__`+type TableDefinedFieldNames<T extends TableSchemaBuilder> = keyof {+ [K in keyof T as K extends `__${string}` ? never : K]: T[K];+};++type Typeof<F extends FieldSchemaBuilder> =+ F extends FieldSchemaBuilder<infer Type> ? Type : never;+type RequiredFields<T extends TableSchemaBuilder> = {- [K in keyof T as T[K] extends FieldSchemaBuilder<any, infer Optional>+ [K in TableDefinedFieldNames<T> as T[K] extends FieldSchemaBuilder<+ any,+ infer Optional+ >? Optional extends false? K: never- : never]: T[K] extends FieldSchemaBuilder<infer Type> ? Type : never;+ : never]: Typeof<T[K]>;};type OptionalFields<T extends TableSchemaBuilder> = {- [K in keyof T as T[K] extends FieldSchemaBuilder<any, infer Optional>+ [K in TableDefinedFieldNames<T> as T[K] extends FieldSchemaBuilder<+ any,+ infer Optional+ >? Optional extends true? K: never- : never]?: T[K] extends FieldSchemaBuilder<infer Type>- ? Type | null- : never;+ : never]?: Typeof<T[K]> | null;};type PrimaryKeyField<T extends TableSchemaBuilder> = {- [K in keyof T]: T[K] extends FieldSchemaBuilder<any, any, infer PrimaryKey>+ [K in TableDefinedFieldNames<T>]: T[K] extends FieldSchemaBuilder<+ any,+ any,+ infer PrimaryKey+ >? PrimaryKey extends true? K: never: never;-}[keyof T];+}[TableDefinedFieldNames<T>];-export type NonPrimaryKeyFields<T extends TableSchemaBuilder> = {- [K in keyof T]: T[K] extends FieldSchemaBuilder<any, any, infer PrimaryKey>+type TableDefinedEntity<T extends TableSchemaBuilder> = Pretty<+ RequiredFields<T> &+ OptionalFields<T> & {+ [PrimaryKey in PrimaryKeyField<T>]: Typeof<T[PrimaryKey]>;+ }+>;++type MaybeDocumentEntityWrapper<Schema, Ty> =+ Schema extends DocumentTableSchemaBuilder+ ? Ty & {+ [key: string]: any;+ }+ : Ty;++type NonPrimaryKeyFieldNames<T extends TableSchemaBuilder> = {+ [K in TableDefinedFieldNames<T>]: T[K] extends FieldSchemaBuilder<+ any,+ any,+ infer PrimaryKey+ >? PrimaryKey extends false? K: never: never;-}[keyof T];+}[TableDefinedFieldNames<T>];-export type PrimaryKeyFieldType<T extends TableSchemaBuilder> =- T[PrimaryKeyField<T>] extends FieldSchemaBuilder<infer Type>- ? Type extends Key- ? Type- : never- : never;+// CRUD api types+export type PrimaryKeyFieldType<T extends TableSchemaBuilder> = Typeof<+ T[PrimaryKeyField<T>]+>;export type CreateEntityInput<T extends TableSchemaBuilder> = Pretty<- RequiredFields<T> & OptionalFields<T>+ MaybeDocumentEntityWrapper<T, RequiredFields<T> & OptionalFields<T>>>;// @TODO(@forehalo): return value need to be specified with `Default` inferenceexport type Entity<T extends TableSchemaBuilder> = Pretty<- CreateEntityInput<T> & {- [key in PrimaryKeyField<T>]: PrimaryKeyFieldType<T>;- }+ MaybeDocumentEntityWrapper<T, TableDefinedEntity<T>>>;-export type UpdateEntityInput<T extends TableSchemaBuilder> = Pretty<{- [key in NonPrimaryKeyFields<T>]?: key extends keyof Entity<T>- ? Entity<T>[key]- : never;-}>;+export type UpdateEntityInput<T extends TableSchemaBuilder> = Pretty<+ MaybeDocumentEntityWrapper<+ T,+ {+ [key in NonPrimaryKeyFieldNames<T>]?: key extends keyof TableDefinedEntity<T>+ ? TableDefinedEntity<T>[key]+ : never;+ }+ >+>;-export type FindEntityInput<T extends TableSchemaBuilder> = Pretty<{- [key in keyof T]?: key extends keyof Entity<T> ? Entity<T>[key] : never;-}>;+export type FindEntityInput<T extends TableSchemaBuilder> = Pretty<+ MaybeDocumentEntityWrapper<+ T,+ {+ [key in TableDefinedFieldNames<T>]?: key extends keyof TableDefinedEntity<T>+ ? TableDefinedEntity<T>[key]+ : never;+ }+ >+>;export class Table<T extends TableSchemaBuilder> {- readonly schema: TableSchema;+ readonly schema: TableSchema = {};readonly keyField: string = '';private readonly adapter: TableAdapter;+ public readonly isDocumentTable: boolean = false;private readonly subscribedKeys: Map<Key, Observable<any>> = new Map();@@ -92,17 +136,20 @@ export class Table<T extends TableSchemaBuilder> {private readonly opts: TableOptions) {this.adapter = db.table(name) as any;- this.schema = Object.entries(this.opts.schema).reduce(- (acc, [fieldName, fieldBuilder]) => {- acc[fieldName] = fieldBuilder.schema;- if (fieldBuilder.schema.isPrimaryKey) {- // @ts-expect-error still in constructor- this.keyField = fieldName;+ for (const [fieldName, fieldBuilder] of Object.entries(this.opts.schema)) {+ // handle internal fields+ if (fieldName.startsWith('__')) {+ if (fieldName === '__document') {+ this.isDocumentTable = true;}- return acc;- },- {} as TableSchema- );+ continue;+ }++ this.schema[fieldName] = fieldBuilder.schema;+ if (fieldBuilder.schema.isPrimaryKey) {+ this.keyField = fieldName;+ }+ }this.adapter.setup({ ...opts, keyField: this.keyField });}@@ -129,7 +176,7 @@ export class Table<T extends TableSchemaBuilder> {validators.validateCreateEntityData(this, data);return this.adapter.insert({- data: data,+ data,});}diff --git a/packages/common/infra/src/orm/core/validators/data.ts b/packages/common/infra/src/orm/core/validators/data.tsindex d3ddf250b0..1c8c9fdd65 100644--- a/packages/common/infra/src/orm/core/validators/data.ts+++ b/packages/common/infra/src/orm/core/validators/data.ts@@ -52,32 +52,33 @@ export const dataValidators = {validate(table, data) {for (const key in data) {const field = table.schema[key];- if (!field) {- throw new Error(- `[Table(${table.name})]: Field '${key}' is not defined but set in entity.`- );- }+ if (field) {+ const val = data[key];- const val = data[key];+ if (val === undefined) {+ delete data[key];+ continue;+ }- if (val === undefined) {- delete data[key];- continue;- }+ if (val === null) {+ if (!field.optional) {+ throw new Error(+ `[Table(${table.name})]: Field '${key}' is required but not set.`+ );+ }+ continue;+ }- if (val === null) {- if (!field.optional) {+ const typeGet = inputType(val);+ if (!typeMatches(field.type, typeGet)) {throw new Error(- `[Table(${table.name})]: Field '${key}' is required but not set.`+ `[Table(${table.name})]: Field '${key}' type mismatch. Expected ${field.type} got ${typeGet}.`);}- continue;- }-- const typeGet = inputType(val);- if (!typeMatches(field.type, typeGet)) {+ } else if (!table.isDocumentTable) {+ // strict check field existence for normal tablethrow new Error(- `[Table(${table.name})]: Field '${key}' type mismatch. Expected ${field.type} got ${typeGet}.`+ `[Table(${table.name})]: Field '${key}' is not defined but set in entity.`);}}@@ -86,33 +87,35 @@ export const dataValidators = {DataTypeShouldExactlyMatch: {validate(table, data) {const keys: Set<string> = new Set();+for (const key in data) {const field = table.schema[key];- if (!field) {- throw new Error(- `[Table(${table.name})]: Field '${key}' is not defined but set in entity.`- );- }-- const val = data[key];+ if (field) {+ const val = data[key];++ if (val === undefined || val === null) {+ if (!field.optional) {+ throw new Error(+ `[Table(${table.name})]: Field '${key}' is required but not set.`+ );+ }+ continue;+ }- if (val === undefined || val === null) {- if (!field.optional) {+ const typeGet = inputType(val);+ if (!typeMatches(field.type, typeGet)) {throw new Error(- `[Table(${table.name})]: Field '${key}' is required but not set.`+ `[Table(${table.name})]: Field '${key}' type mismatch. Expected type '${field.type}' but got '${typeGet}'.`);}- continue;- }- const typeGet = inputType(val);- if (!typeMatches(field.type, typeGet)) {+ keys.add(key);+ } else if (!table.isDocumentTable) {+ // strict check field existence for normal tablethrow new Error(- `[Table(${table.name})]: Field '${key}' type mismatch. Expected type '${field.type}' but got '${typeGet}'.`+ `[Table(${table.name})]: Field '${key}' is not defined but set in entity.`);}-- keys.add(key);}for (const key in table.schema) {