Skip to content

There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.

feat(infra): better orm (#7502)

EYHN9fe77baf0501
diff --git a/packages/common/infra/src/orm/core/__tests__/yjs.spec.ts b/packages/common/infra/src/orm/core/__tests__/yjs.spec.ts
index 07c27ab014..2a78ca3ebd 100644
--- a/packages/common/infra/src/orm/core/__tests__/yjs.spec.ts
+++ b/packages/common/infra/src/orm/core/__tests__/yjs.spec.ts
@@ -26,6 +26,7 @@ const TEST_SCHEMA = {
users: {
id: f.number().primaryKey().default(incremental()),
name: f.string(),
+ email: f.string().optional(),
},
} satisfies DBSchemaBuilder;
@@ -211,9 +212,11 @@ describe('ORM entity CRUD', () => {
test('should be able to subscribe to entity key list', t => {
const { client } = t;
+ let callbackCount = 0;
let keys: string[] = [];
const subscription = client.tags.keys$().subscribe(data => {
keys = data;
+ callbackCount++;
});
client.tags.create({
@@ -229,6 +232,7 @@ describe('ORM entity CRUD', () => {
client.tags.delete('test');
expect(keys).toStrictEqual([]);
+ expect(callbackCount).toStrictEqual(3); // init, create, delete
subscription.unsubscribe();
});
@@ -236,9 +240,11 @@ describe('ORM entity CRUD', () => {
test('should be able to subscribe to filtered entity changes', t => {
const { client } = t;
+ let callbackCount = 0;
let entities: any[] = [];
const subscription = client.tags.find$({ name: 'test' }).subscribe(data => {
entities = data;
+ callbackCount++;
});
const tag1 = client.tags.create({
@@ -257,6 +263,24 @@ describe('ORM entity CRUD', () => {
expect(entities).toStrictEqual([tag1, tag2]);
+ client.tags.create({
+ id: '3',
+ name: 'not-test',
+ color: 'yellow',
+ });
+
+ expect(entities).toStrictEqual([tag1, tag2]);
+ expect(callbackCount).toStrictEqual(3);
+
+ client.tags.update('1', { color: 'green' });
+ expect(entities).toStrictEqual([{ ...tag1, color: 'green' }, tag2]);
+
+ client.tags.delete('1');
+ expect(entities).toStrictEqual([tag2]);
+
+ client.tags.delete('2');
+ expect(entities).toStrictEqual([]);
+
subscription.unsubscribe();
});
@@ -264,7 +288,7 @@ describe('ORM entity CRUD', () => {
const { client } = t;
let entities: any[] = [];
- const subscription = client.tags.find$({}).subscribe(data => {
+ const subscription = client.tags.find$().subscribe(data => {
entities = data;
});
@@ -302,4 +326,82 @@ describe('ORM entity CRUD', () => {
"[Table(tags)]: Field '$$DELETED' is reserved keyword and can't be used"
);
});
+
+ test('should be able to validate entity data', t => {
+ const { client } = t;
+
+ expect(() => {
+ client.users.create({
+ // @ts-expect-error
+ name: null,
+ });
+ }).toThrowError("Field 'name' is required but not set.");
+
+ expect(() => {
+ // @ts-expect-error
+ client.users.create({});
+ }).toThrowError("Field 'name' is required but not set.");
+
+ expect(() => {
+ client.users.update(1, {
+ // @ts-expect-error
+ name: null,
+ });
+ }).toThrowError("Field 'name' is required but not set.");
+ });
+
+ test('should be able to set optional field to null', t => {
+ const { client } = t;
+
+ {
+ const user = client.users.create({
+ name: 'test',
+ });
+
+ expect(user.email).toBe(null);
+ }
+
+ {
+ const user = client.users.create({
+ name: 'test',
+ email: null,
+ });
+
+ expect(user.email).toBe(null);
+ }
+
+ {
+ const user = client.users.create({
+ name: 'test',
+ email: '[email protected]',
+ });
+
+ client.users.update(user.id, {
+ email: null,
+ });
+
+ expect(client.users.get(user.id)!.email).toBe(null);
+ }
+ });
+
+ test('should be able to find entity by optional field', t => {
+ const { client } = t;
+
+ const user = client.users.create({
+ name: 'test',
+ email: null,
+ });
+
+ {
+ const found = client.users.find({ email: null });
+
+ expect(found).toEqual([user]);
+ }
+
+ {
+ const found = client.users.find({ email: undefined });
+
+ expect(found).toEqual([]);
+ }
+ });
});
diff --git a/packages/common/infra/src/orm/core/adapters/yjs/table.ts b/packages/common/infra/src/orm/core/adapters/yjs/table.ts
index c1ef9e1c98..712d646c71 100644
--- a/packages/common/infra/src/orm/core/adapters/yjs/table.ts
+++ b/packages/common/infra/src/orm/core/adapters/yjs/table.ts
@@ -104,36 +104,47 @@ export class YjsTableAdapter implements TableAdapter {
const { where, select, callback } = query;
let listeningOnAll = false;
- const obKeys = new Set<any>();
- const results = [];
+ const results = new Map<string, any>();
if (!where) {
listeningOnAll = true;
- } else if ('byKey' in where) {
- obKeys.add(where.byKey.toString());
}
for (const record of this.iterate(where)) {
- if (!listeningOnAll) {
- obKeys.add(this.keyof(record));
- }
- results.push(this.value(record, select));
+ results.set(this.keyof(record), this.value(record, select));
}
- callback(results);
+ callback(Array.from(results.values()));
const ob = (tx: Transaction) => {
+ let hasChanged = false;
for (const [ty] of tx.changed) {
- const record = ty as unknown as AbstractType<any>;
- if (
- listeningOnAll ||
- obKeys.has(this.keyof(record)) ||
- (where && this.match(record, where))
- ) {
- callback(this.find({ where, select }));
- return;
+ const record = ty;
+ const key = this.keyof(record);
+ const isMatch =
+ (listeningOnAll || (where && this.match(record, where))) &&
+ !this.isDeleted(record);
+ const prevMatch = results.get(key);
+ const isPrevMatched = results.has(key);
+
+ if (isMatch && isPrevMatched) {
+ const newValue = this.value(record, select);
+ if (prevMatch !== newValue) {
+ results.set(key, newValue);
+ hasChanged = true;
+ }
+ } else if (isMatch && !isPrevMatched) {
+ results.set(this.keyof(record), this.value(record, select));
+ hasChanged = true;
+ } else if (!isMatch && isPrevMatched) {
+ results.delete(key);
+ hasChanged = true;
}
}
+
+ if (hasChanged) {
+ callback(Array.from(results.values()));
+ }
};
this.doc.on('afterTransaction', ob);
@@ -165,9 +176,16 @@ export class YjsTableAdapter implements TableAdapter {
return null;
}
- private *iterate(where: WhereCondition = []) {
+ private *iterate(where?: WhereCondition) {
+ if (!where) {
+ for (const map of this.doc.share.values()) {
+ if (!this.isDeleted(map)) {
+ yield map;
+ }
+ }
+ }
// fast pass for key lookup without iterating the whole table
- if ('byKey' in where) {
+ else if ('byKey' in where) {
const record = this.recordByKey(where.byKey.toString());
if (record) {
yield record;
@@ -202,7 +220,9 @@ export class YjsTableAdapter implements TableAdapter {
return (
!this.isDeleted(record) &&
(Array.isArray(where)
- ? where.every(c => this.field(record, c.field) === c.value)
+ ? where.length === 0
+ ? false
+ : where.every(c => this.field(record, c.field) === c.value)
: where.byKey === this.keyof(record))
);
}
diff --git a/packages/common/infra/src/orm/core/table.ts b/packages/common/infra/src/orm/core/table.ts
index 1fbf908ba7..0fc6468b58 100644
--- a/packages/common/infra/src/orm/core/table.ts
+++ b/packages/common/infra/src/orm/core/table.ts
@@ -30,7 +30,9 @@ type OptionalFields<T extends TableSchemaBuilder> = {
? Optional extends true
? K
: never
- : never]?: T[K] extends FieldSchemaBuilder<infer Type> ? Type : never;
+ : never]?: T[K] extends FieldSchemaBuilder<infer Type>
+ ? Type | null
+ : never;
};
type PrimaryKeyField<T extends TableSchemaBuilder> = {
@@ -68,17 +70,13 @@ export type Entity<T extends TableSchemaBuilder> = Pretty<
>;
export type UpdateEntityInput<T extends TableSchemaBuilder> = Pretty<{
- [key in NonPrimaryKeyFields<T>]?: T[key] extends FieldSchemaBuilder<
- infer Type
- >
- ? Type
+ [key in NonPrimaryKeyFields<T>]?: key extends keyof Entity<T>
+ ? Entity<T>[key]
: never;
}>;
export type FindEntityInput<T extends TableSchemaBuilder> = Pretty<{
- [key in keyof T]?: T[key] extends FieldSchemaBuilder<infer Type>
- ? Type
- : never;
+ [key in keyof T]?: key extends keyof Entity<T> ? Entity<T>[key] : never;
}>;
export class Table<T extends TableSchemaBuilder> {
@@ -192,22 +190,30 @@ export class Table<T extends TableSchemaBuilder> {
return ob$;
}
- find(where: FindEntityInput<T>): Entity<T>[] {
+ find(where?: FindEntityInput<T>): Entity<T>[] {
return this.adapter.find({
- where: Object.entries(where).map(([field, value]) => ({
- field,
- value,
- })),
+ where: !where
+ ? undefined
+ : Object.entries(where)
+ .map(([field, value]) => ({
+ field,
+ value,
+ }))
+ .filter(({ value }) => value !== undefined),
});
}
- find$(where: FindEntityInput<T>): Observable<Entity<T>[]> {
+ find$(where?: FindEntityInput<T>): Observable<Entity<T>[]> {
return new Observable<Entity<T>[]>(subscriber => {
const unsubscribe = this.adapter.observe({
- where: Object.entries(where).map(([field, value]) => ({
- field,
- value,
- })),
+ where: !where
+ ? undefined
+ : Object.entries(where)
+ .map(([field, value]) => ({
+ field,
+ value,
+ }))
+ .filter(({ value }) => value !== undefined),
callback: data => {
subscriber.next(data);
},
diff --git a/packages/common/infra/src/orm/core/validators/data.ts b/packages/common/infra/src/orm/core/validators/data.ts
index db176c1173..d3ddf250b0 100644
--- a/packages/common/infra/src/orm/core/validators/data.ts
+++ b/packages/common/infra/src/orm/core/validators/data.ts
@@ -65,14 +65,13 @@ export const dataValidators = {
continue;
}
- if (
- val === null &&
- (!field.optional ||
- field.optional) /* say 'null' can be stored as 'json' */
- ) {
- throw new Error(
- `[Table(${table.name})]: Field '${key}' is required but set as null.`
- );
+ if (val === null) {
+ if (!field.optional) {
+ throw new Error(
+ `[Table(${table.name})]: Field '${key}' is required but not set.`
+ );
+ }
+ continue;
}
const typeGet = inputType(val);
@@ -97,10 +96,13 @@ export const dataValidators = {
const val = data[key];
- if ((val === undefined || val === null) && !field.optional) {
- throw new Error(
- `[Table(${table.name})]: Field '${key}' is required but not set.`
- );
+ if (val === undefined || val === null) {
+ if (!field.optional) {
+ throw new Error(
+ `[Table(${table.name})]: Field '${key}' is required but not set.`
+ );
+ }
+ continue;
}
const typeGet = inputType(val);