There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
feat(infra): opti indexer performance (#8557)
EYHN6ecdc8db7a26
diff --git a/packages/common/infra/src/sync/indexer/__tests__/black-box.spec.ts b/packages/common/infra/src/sync/indexer/__tests__/black-box.spec.tsindex 110f0d286c..0f48aa04b7 100644--- a/packages/common/infra/src/sync/indexer/__tests__/black-box.spec.ts+++ b/packages/common/infra/src/sync/indexer/__tests__/black-box.spec.ts@@ -454,101 +454,107 @@ describe.each([});});- test('subscribe', async () => {- await writeData({- '1': {- title: 'hello world',- },- });-- let value = null as any;- index- .search$({- type: 'match',- field: 'title',- match: 'hello world',- })- .pipe(map(v => (value = v)))- .subscribe();-- await vitest.waitFor(- () => {- expect(value).toEqual({- nodes: [- {- id: '1',- score: expect.anything(),+ test(+ 'subscribe',+ {+ timeout: 30000,+ },+ async () => {+ await writeData({+ '1': {+ title: 'hello world',+ },+ });++ let value = null as any;+ index+ .search$({+ type: 'match',+ field: 'title',+ match: 'hello world',+ })+ .pipe(map(v => (value = v)))+ .subscribe();++ await vitest.waitFor(+ () => {+ expect(value).toEqual({+ nodes: [+ {+ id: '1',+ score: expect.anything(),+ },+ ],+ pagination: {+ count: 1,+ hasMore: false,+ limit: expect.anything(),+ skip: 0,},- ],- pagination: {- count: 1,- hasMore: false,- limit: expect.anything(),- skip: 0,- },- });- },- {- timeout: 5000,- }- );-- await writeData({- '2': {- title: 'hello world',- },- });+ });+ },+ {+ timeout: 10000,+ }+ );- await vitest.waitFor(- () => {- expect(value).toEqual({- nodes: [- {- id: '1',- score: expect.anything(),- },- {- id: '2',- score: expect.anything(),+ await writeData({+ '2': {+ title: 'hello world',+ },+ });++ await vitest.waitFor(+ () => {+ expect(value).toEqual({+ nodes: [+ {+ id: '1',+ score: expect.anything(),+ },+ {+ id: '2',+ score: expect.anything(),+ },+ ],+ pagination: {+ count: 2,+ hasMore: false,+ limit: expect.anything(),+ skip: 0,},- ],- pagination: {- count: 2,- hasMore: false,- limit: expect.anything(),- skip: 0,- },- });- },- {- timeout: 5000,- }- );-- const writer = await index.write();- writer.delete('1');- await writer.commit();-- await vitest.waitFor(- () => {- expect(value).toEqual({- nodes: [- {- id: '2',- score: expect.anything(),+ });+ },+ {+ timeout: 10000,+ }+ );++ const writer = await index.write();+ writer.delete('1');+ await writer.commit();++ await vitest.waitFor(+ () => {+ expect(value).toEqual({+ nodes: [+ {+ id: '2',+ score: expect.anything(),+ },+ ],+ pagination: {+ count: 1,+ hasMore: false,+ limit: expect.anything(),+ skip: 0,},- ],- pagination: {- count: 1,- hasMore: false,- limit: expect.anything(),- skip: 0,- },- });- },- {- timeout: 5000,- }- );- });+ });+ },+ {+ timeout: 10000,+ }+ );+ }+ );});diff --git a/packages/common/infra/src/sync/indexer/impl/indexeddb/data-struct.ts b/packages/common/infra/src/sync/indexer/impl/indexeddb/data-struct.tsindex 2fd5c031a3..9fe55bc1ec 100644--- a/packages/common/infra/src/sync/indexer/impl/indexeddb/data-struct.ts+++ b/packages/common/infra/src/sync/indexer/impl/indexeddb/data-struct.ts@@ -100,7 +100,7 @@ export class DataStruct {}}- async insert(trx: DataStructRWTransaction, document: Document) {+ private async insert(trx: DataStructRWTransaction, document: Document) {const exists = await trx.objectStore('records').index('id')@@ -138,7 +138,7 @@ export class DataStruct {}}- async delete(trx: DataStructRWTransaction, id: string) {+ private async delete(trx: DataStructRWTransaction, id: string) {const nid = await trx.objectStore('records').index('id').getKey(id);if (nid) {@@ -159,11 +159,30 @@ export class DataStruct {deletes: string[],inserts: Document[]) {- for (const del of deletes) {- await this.delete(trx, del);- }- for (const inst of inserts) {- await this.insert(trx, inst);+ const startTime = performance.now();+ try {+ for (const del of deletes) {+ await this.delete(trx, del);+ }+ for (const inst of inserts) {+ await this.insert(trx, inst);+ }+ } finally {+ const endTime = performance.now();+ if (BUILD_CONFIG.debug) {+ performance.measure(+ `[IndexedDB Indexer] Batch Write (${this.databaseName})`,+ {+ start: startTime,+ end: endTime,+ }+ );+ }+ logger.debug(+ `[indexer ${this.databaseName}] batchWrite`,+ endTime - startTime,+ 'ms'+ );}}@@ -214,18 +233,6 @@ export class DataStruct {throw new Error(`Query type '${query.type}' not supported`);}- private async query(- trx: DataStructROTransaction,- query: Query<any>- ): Promise<Match> {- const match = await this.queryRaw(trx, query);- const filteredMatch = match.asyncFilter(async nid => {- const record = await trx.objectStore('records').getKey(nid);- return record !== undefined;- });- return filteredMatch;- }-async clear(trx: DataStructRWTransaction) {await trx.objectStore('records').clear();await trx.objectStore('invertedIndex').clear();@@ -244,7 +251,7 @@ export class DataStruct {limit: options.pagination?.limit ?? 100,};- const match = await this.query(trx, query);+ const match = await this.queryRaw(trx, query);const nids = match.toArray()@@ -252,7 +259,11 @@ export class DataStruct {const nodes = [];for (const nid of nids) {- nodes.push(await this.resultNode(trx, match, nid, options));+ const record = await trx.objectStore('records').get(nid);+ if (!record) {+ continue;+ }+ nodes.push(this.resultNode(record, options, match, nid));}return {@@ -265,9 +276,20 @@ export class DataStruct {nodes: nodes,};} finally {+ const endTime = performance.now();+ if (BUILD_CONFIG.debug) {+ performance.measure(+ `[IndexedDB Indexer] Search (${this.databaseName})`,+ {+ detail: { query, options },+ start: startTime,+ end: endTime,+ }+ );+ }logger.debug(`[indexer ${this.databaseName}] search`,- performance.now() - startTime,+ endTime - startTime,'ms',query);@@ -297,7 +319,7 @@ export class DataStruct {limit: 0,};- const match = await this.query(trx, query);+ const match = await this.queryRaw(trx, query);const nids = match.toArray();@@ -308,9 +330,11 @@ export class DataStruct {}[] = [];for (const nid of nids) {- const values = (await trx.objectStore('records').get(nid))?.data.get(- field- );+ const record = await trx.objectStore('records').get(nid);+ if (!record) {+ continue;+ }+ const values = record.data.get(field);for (const value of values ?? []) {let bucket;let bucketIndex = buckets.findIndex(b => b.key === value);@@ -332,7 +356,7 @@ export class DataStruct {bucket.nids.length - 1 < hitPagination.skip + hitPagination.limit) {bucket.hits.push(- await this.resultNode(trx, match, nid, options.hits ?? {})+ this.resultNode(record, options.hits ?? {}, match, nid));}}@@ -373,9 +397,20 @@ export class DataStruct {},};} finally {+ const endTime = performance.now();+ if (BUILD_CONFIG.debug) {+ performance.measure(+ `[IndexedDB Indexer] Aggregate (${this.databaseName})`,+ {+ detail: { query, field, options },+ start: startTime,+ end: endTime,+ }+ );+ }logger.debug(`[indexer ${this.databaseName}] aggregate`,- performance.now() - startTime,+ endTime - startTime,'ms');}@@ -383,12 +418,19 @@ export class DataStruct {async getAll(trx: DataStructROTransaction,- ids: string[]+ ids?: string[]): Promise<Document[]> {const docs = [];- for (const id of ids) {- const record = await trx.objectStore('records').index('id').get(id);- if (record) {+ if (ids) {+ for (const id of ids) {+ const record = await trx.objectStore('records').index('id').get(id);+ if (record) {+ docs.push(Document.from(record.id, record.data));+ }+ }+ } else {+ const records = await trx.objectStore('records').getAll();+ for (const record of records) {docs.push(Document.from(record.id, record.data));}}@@ -405,7 +447,10 @@ export class DataStruct {await this.ensureInitialized();return this.database.transaction(['records', 'invertedIndex', 'kvMetadata'],- 'readonly'+ 'readonly',+ {+ durability: 'relaxed',+ });}@@ -413,7 +458,10 @@ export class DataStruct {await this.ensureInitialized();return this.database.transaction(['records', 'invertedIndex', 'kvMetadata'],- 'readwrite'+ 'readwrite',+ {+ durability: 'relaxed',+ });}@@ -446,20 +494,15 @@ export class DataStruct {});}- private async resultNode(- trx: DataStructROTransaction,- match: Match,- nid: number,- options: SearchOptions<any>- ): Promise<SearchResult<any, any>['nodes'][number]> {- const record = await trx.objectStore('records').get(nid);- if (!record) {- throw new Error(`Record not found for nid ${nid}`);- }-+ private resultNode(+ record: { id: string; data: Map<string, string[]> },+ options: SearchOptions<any>,+ match?: Match,+ nid?: number+ ): SearchResult<any, any>['nodes'][number] {const node = {id: record.id,- score: match.getScore(nid),+ score: match && nid ? match.getScore(nid) : 1,} as any;if (options.fields) {@@ -473,7 +516,7 @@ export class DataStruct {node.fields = fields;}- if (options.highlights) {+ if (match && nid && options.highlights) {const highlights = {} as Record<string, string[]>;for (const { field, before, end } of options.highlights) {const highlightValues = match.getHighlighters(nid, field);diff --git a/packages/common/infra/src/sync/indexer/impl/indexeddb/index.ts b/packages/common/infra/src/sync/indexer/impl/indexeddb/index.tsindex b9244bee01..2cbca87a5b 100644--- a/packages/common/infra/src/sync/indexer/impl/indexeddb/index.ts+++ b/packages/common/infra/src/sync/indexer/impl/indexeddb/index.ts@@ -61,7 +61,7 @@ export class IndexedDBIndex<S extends Schema> implements Index<S> {options: SearchOptions<any> = {}): Observable<SearchResult<any, SearchOptions<any>>> {return merge(of(1), this.broadcast$).pipe(- throttleTime(500, undefined, { leading: true, trailing: true }),+ throttleTime(3000, undefined, { leading: true, trailing: true }),exhaustMapWithTrailing(() => {return from((async () => {@@ -88,7 +88,7 @@ export class IndexedDBIndex<S extends Schema> implements Index<S> {options: AggregateOptions<any> = {}): Observable<AggregateResult<S, AggregateOptions<any>>> {return merge(of(1), this.broadcast$).pipe(- throttleTime(500, undefined, { leading: true, trailing: true }),+ throttleTime(3000, undefined, { leading: true, trailing: true }),exhaustMapWithTrailing(() => {return from((async () => {@@ -120,7 +120,7 @@ export class IndexedDBIndexWriter<S extends Schema> implements IndexWriter<S> {return (await this.getAll([id]))[0] ?? null;}- async getAll(ids: string[]): Promise<Document<S>[]> {+ async getAll(ids?: string[]): Promise<Document<S>[]> {const trx = await this.data.readonly();return this.data.getAll(trx, ids);}@@ -138,6 +138,7 @@ export class IndexedDBIndexWriter<S extends Schema> implements IndexWriter<S> {async commit(): Promise<void> {await this.data.batchWrite(this.trx, this.deletes, this.inserts);+ this.trx.commit();this.channel.postMessage(1);}diff --git a/packages/common/infra/src/sync/indexer/impl/indexeddb/inverted-index.ts b/packages/common/infra/src/sync/indexer/impl/indexeddb/inverted-index.tsindex 4230e18dfc..135f8ec631 100644--- a/packages/common/infra/src/sync/indexer/impl/indexeddb/inverted-index.ts+++ b/packages/common/infra/src/sync/indexer/impl/indexeddb/inverted-index.ts@@ -202,6 +202,12 @@ export class FullTextInvertedIndex implements InvertedIndex {}>>();+ const avgFieldLength =+ (+ await trx+ .objectStore('kvMetadata')+ .get(`full-text:avg-field-length:${this.fieldKey}`)+ )?.value ?? 0;for (const token of queryTokens) {const key = InvertedIndexKey.forString(this.fieldKey, token.term);const objs = await trx@@ -229,12 +235,6 @@ export class FullTextInvertedIndex implements InvertedIndex {};const termFreq = position.rs.length;const totalCount = objs.length;- const avgFieldLength =- (- await trx- .objectStore('kvMetadata')- .get(`full-text:avg-field-length:${this.fieldKey}`)- )?.value ?? 0;const fieldLength = position.l;const score =bm25(termFreq, 1, totalCount, fieldLength, avgFieldLength) *diff --git a/packages/common/infra/src/sync/indexer/impl/indexeddb/match.ts b/packages/common/infra/src/sync/indexer/impl/indexeddb/match.tsindex 10dbce0ca2..707779a2ea 100644--- a/packages/common/infra/src/sync/indexer/impl/indexeddb/match.ts+++ b/packages/common/infra/src/sync/indexer/impl/indexeddb/match.ts@@ -95,28 +95,6 @@ export class Match {.map(e => e[0]);}- filter(predicate: (id: number) => boolean) {- const newWeight = new Match();- for (const [id, score] of this.scores) {- if (predicate(id)) {- newWeight.addScore(id, score);- newWeight.copyExtData(this, id);- }- }- return newWeight;- }-- async asyncFilter(predicate: (id: number) => Promise<boolean>) {- const newWeight = new Match();- for (const [id, score] of this.scores) {- if (await predicate(id)) {- newWeight.addScore(id, score);- newWeight.copyExtData(this, id);- }- }- return newWeight;- }-private copyExtData(from: Match, id: number) {for (const [field, values] of from.highlighters.get(id) ?? []) {for (const [index, ranges] of values) {diff --git a/packages/common/infra/src/sync/indexer/impl/memory/data-struct.ts b/packages/common/infra/src/sync/indexer/impl/memory/data-struct.tsindex 96a77812f7..a22ced413c 100644--- a/packages/common/infra/src/sync/indexer/impl/memory/data-struct.ts+++ b/packages/common/infra/src/sync/indexer/impl/memory/data-struct.ts@@ -47,16 +47,22 @@ export class DataStruct {}}- getAll(ids: string[]): Document[] {- return ids- .map(id => {- const nid = this.idMap.get(id);- if (nid === undefined) {- return undefined;- }- return Document.from(id, this.records[nid].data);- })- .filter((v): v is Document => v !== undefined);+ getAll(ids?: string[]): Document[] {+ if (ids) {+ return ids+ .map(id => {+ const nid = this.idMap.get(id);+ if (nid === undefined) {+ return undefined;+ }+ return Document.from(id, this.records[nid].data);+ })+ .filter((v): v is Document => v !== undefined);+ } else {+ return this.records+ .filter(record => !record.deleted)+ .map(record => Document.from(record.id, record.data));+ }}insert(document: Document) {diff --git a/packages/common/infra/src/sync/indexer/impl/memory/index.ts b/packages/common/infra/src/sync/indexer/impl/memory/index.tsindex 2d05557cde..c3bddd9dac 100644--- a/packages/common/infra/src/sync/indexer/impl/memory/index.ts+++ b/packages/common/infra/src/sync/indexer/impl/memory/index.ts@@ -28,7 +28,7 @@ export class MemoryIndex<S extends Schema> implements Index<S> {return (await this.getAll([id]))[0] ?? null;}- getAll(ids: string[]): Promise<Document<S>[]> {+ getAll(ids?: string[]): Promise<Document<S>[]> {return Promise.resolve(this.data.getAll(ids));}diff --git a/packages/common/infra/src/sync/indexer/indexer.ts b/packages/common/infra/src/sync/indexer/indexer.tsindex cba453022f..843ed36655 100644--- a/packages/common/infra/src/sync/indexer/indexer.ts+++ b/packages/common/infra/src/sync/indexer/indexer.ts@@ -31,7 +31,7 @@ export interface IndexWriter<S extends Schema>export interface IndexReader<S extends Schema> {get(id: string): Promise<Document<S> | null>;- getAll(ids: string[]): Promise<Document<S>[]>;+ getAll(ids?: string[]): Promise<Document<S>[]>;has(id: string): Promise<boolean>;}diff --git a/packages/common/infra/src/sync/job/impl/indexeddb/index.ts b/packages/common/infra/src/sync/job/impl/indexeddb/index.tsindex 20f6f8e471..f6215d52d0 100644--- a/packages/common/infra/src/sync/job/impl/indexeddb/index.ts+++ b/packages/common/infra/src/sync/job/impl/indexeddb/index.ts@@ -50,7 +50,9 @@ export class IndexedDBJobQueue<J> implements JobQueue<J> {async accept(): Promise<Job[] | null> {await this.ensureInitialized();const jobs = [];- const trx = this.database.transaction(['jobs'], 'readwrite');+ const trx = this.database.transaction(['jobs'], 'readwrite', {+ durability: 'relaxed',+ });// if no priority jobs@@ -148,7 +150,9 @@ export class IndexedDBJobQueue<J> implements JobQueue<J> {async complete(jobs: Job[]): Promise<void> {await this.ensureInitialized();- const trx = this.database.transaction(['jobs'], 'readwrite');+ const trx = this.database.transaction(['jobs'], 'readwrite', {+ durability: 'relaxed',+ });for (const { id } of jobs) {await trx@@ -162,7 +166,9 @@ export class IndexedDBJobQueue<J> implements JobQueue<J> {async return(jobs: Job[], retry: boolean = false): Promise<void> {await this.ensureInitialized();- const trx = this.database.transaction(['jobs'], 'readwrite');+ const trx = this.database.transaction(['jobs'], 'readwrite', {+ durability: 'relaxed',+ });for (const { id } of jobs) {if (retry) {@@ -185,7 +191,9 @@ export class IndexedDBJobQueue<J> implements JobQueue<J> {async clear(): Promise<void> {await this.ensureInitialized();- const trx = this.database.transaction(['jobs'], 'readwrite');+ const trx = this.database.transaction(['jobs'], 'readwrite', {+ durability: 'relaxed',+ });await trx.objectStore('jobs').clear();}diff --git a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.tsindex 19a99e4825..1c81110d5c 100644--- a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts+++ b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts@@ -139,19 +139,7 @@ export class DocsIndexer extends Entity {return;}- const allIndexedDocs = (- await this.docIndex.search(- {- type: 'all',- },- {- pagination: {- limit: Number.MAX_SAFE_INTEGER,- skip: 0,- },- }- )- ).nodes.map(n => n.id);+ const allIndexedDocs = (await this.docIndex.getAll()).map(d => d.id);workerOutput = await worker.run({type: 'rootDoc',diff --git a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts b/packages/frontend/core/src/modules/docs-search/worker/in-worker.tsindex 21d03eccd1..49e6982e09 100644--- a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts+++ b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts@@ -20,7 +20,10 @@ import type {WorkerOutput,} from './types';-let cachedRootDoc: { doc: YDoc; hash: string } | null = null;+const LRU_CACHE_SIZE = 5;++// lru cache for ydoc instances, last used at the end of the array+const lruCache = [] as { doc: YDoc; hash: string }[];async function digest(data: Uint8Array) {if (@@ -35,6 +38,29 @@ async function digest(data: Uint8Array) {return lib0Digest(data);}+async function getOrCreateCachedYDoc(data: Uint8Array) {+ try {+ const hash = toHexString(await digest(data));+ const cachedIndex = lruCache.findIndex(item => item.hash === hash);+ if (cachedIndex !== -1) {+ const cached = lruCache.splice(cachedIndex, 1)[0];+ lruCache.push(cached);+ return cached.doc;+ } else {+ const doc = new YDoc();+ if (!isEmptyUpdate(data)) {+ applyUpdate(doc, data);+ }+ lruCache.push({ doc, hash });+ return doc;+ }+ } finally {+ if (lruCache.length > LRU_CACHE_SIZE) {+ lruCache.shift();+ }+ }+}+async function crawlingDocData({docBuffer,storageDocId,@@ -45,16 +71,7 @@ async function crawlingDocData({return {};}- const rootDocBufferHash = toHexString(await digest(rootDocBuffer));-- let yRootDoc;- if (cachedRootDoc && cachedRootDoc.hash === rootDocBufferHash) {- yRootDoc = cachedRootDoc.doc;- } else {- yRootDoc = new YDoc();- applyUpdate(yRootDoc, rootDocBuffer);- cachedRootDoc = { doc: yRootDoc, hash: rootDocBufferHash };- }+ const yRootDoc = await getOrCreateCachedYDoc(rootDocBuffer);let docId = null;for (const [id, subdoc] of yRootDoc.getMap('spaces')) {@@ -83,16 +100,18 @@ async function crawlingDocData({deletedDoc: [docId],};} else {- const ydoc = new YDoc();+ if (isEmptyUpdate(docBuffer)) {+ return {+ deletedDoc: [docId],+ };+ }++ const ydoc = await getOrCreateCachedYDoc(docBuffer);let docTitle = '';let summaryLenNeeded = 1000;let summary = '';const blockDocuments: Document<BlockIndexSchema>[] = [];- if (!isEmptyUpdate(docBuffer)) {- applyUpdate(ydoc, docBuffer);- }-const blocks = ydoc.getMap<any>('blocks');if (blocks.size === 0) {@@ -363,16 +382,14 @@ async function crawlingDocData({}}-function crawlingRootDocData({+async function crawlingRootDocData({allIndexedDocs,rootDocBuffer,reindexAll,}: WorkerInput & {type: 'rootDoc';-}): WorkerOutput {- const ydoc = new YDoc();-- applyUpdate(ydoc, rootDocBuffer);+}): Promise<WorkerOutput> {+ const ydoc = await getOrCreateCachedYDoc(rootDocBuffer);const docs = ydoc.getMap('meta').get('pages') as| YArray<YMap<any>>@@ -422,7 +439,7 @@ globalThis.onmessage = async (event: MessageEvent<WorkerIngoingMessage>) => {try {let data;if (input.type === 'rootDoc') {- data = crawlingRootDocData(input);+ data = await crawlingRootDocData(input);} else {data = await crawlingDocData(input);}