There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
feat(core): indexer upgrade (#8062)
EYHNfb64bc7e55a8
diff --git a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.tsindex e506ec2e63..a54e304f31 100644--- a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts+++ b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts@@ -1,5 +1,10 @@import { DebugLogger } from '@affine/debug';-import type { Job, JobQueue, WorkspaceService } from '@toeverything/infra';+import type {+ Job,+ JobQueue,+ WorkspaceLocalState,+ WorkspaceService,+} from '@toeverything/infra';import {Entity,IndexedDBIndexStorage,@@ -21,12 +26,18 @@ export function isEmptyUpdate(binary: Uint8Array) {}const logger = new DebugLogger('crawler');+const WORKSPACE_DOCS_INDEXER_VERSION_KEY = 'docs-indexer-version';interface IndexerJobPayload {storageDocId: string;}export class DocsIndexer extends Entity {+ /**+ * increase this number to re-index all docs+ */+ static INDEXER_VERSION = 1;+private readonly jobQueue: JobQueue<IndexerJobPayload> =new IndexedDBJobQueue<IndexerJobPayload>('jq:' + this.workspaceService.workspace.id@@ -66,7 +77,10 @@ export class DocsIndexer extends Entity {{});- constructor(private readonly workspaceService: WorkspaceService) {+ constructor(+ private readonly workspaceService: WorkspaceService,+ private readonly workspaceLocalState: WorkspaceLocalState+ ) {super();}@@ -96,6 +110,16 @@ export class DocsIndexer extends Entity {return;}+ const dbVersion = this.getVersion();++ if (dbVersion > DocsIndexer.INDEXER_VERSION) {+ // stop if db version is higher then self+ this.runner.stop();+ throw new Error('Indexer is outdated');+ }++ const isUpgrade = dbVersion < DocsIndexer.INDEXER_VERSION;+// jobs should have the same storage docId, so we just pick the first oneconst storageDocId = jobs[0].payload.storageDocId;@@ -111,7 +135,6 @@ export class DocsIndexer extends Entity {await this.workspaceEngine.doc.storage.loadDocFromLocal(this.workspaceId);-if (!rootDocBuffer) {return;}@@ -134,6 +157,7 @@ export class DocsIndexer extends Entity {type: 'rootDoc',allIndexedDocs,rootDocBuffer,+ reindexAll: isUpgrade,});} else {const rootDocBuffer =@@ -226,6 +250,10 @@ export class DocsIndexer extends Entity {);}+ if (isUpgrade) {+ this.setVersion();+ }+const duration = performance.now() - startTime;logger.debug('Finish crawling job for storageDocId:' +@@ -238,6 +266,7 @@ export class DocsIndexer extends Entity {startCrawling() {this.runner.start();+this.jobQueue.enqueue([{@@ -257,6 +286,27 @@ export class DocsIndexer extends Entity {return this.worker;}+ getVersion() {+ const version = this.workspaceLocalState.get<number>(+ WORKSPACE_DOCS_INDEXER_VERSION_KEY+ );+ if (typeof version !== 'number') {+ return -1;+ } else {+ return version;+ }+ }++ setVersion(version = DocsIndexer.INDEXER_VERSION) {+ if (this.getVersion() >= version) {+ return;+ }+ return this.workspaceLocalState.set(+ WORKSPACE_DOCS_INDEXER_VERSION_KEY,+ version+ );+ }+override dispose(): void {this.runner.stop();}diff --git a/packages/frontend/core/src/modules/docs-search/index.ts b/packages/frontend/core/src/modules/docs-search/index.tsindex 19df647ba9..03fd0e157a 100644--- a/packages/frontend/core/src/modules/docs-search/index.ts+++ b/packages/frontend/core/src/modules/docs-search/index.ts@@ -2,6 +2,7 @@ export { DocsSearchService } from './services/docs-search';import {type Framework,+ WorkspaceLocalState,WorkspaceScope,WorkspaceService,} from '@toeverything/infra';@@ -13,5 +14,5 @@ export function configureDocsSearchModule(framework: Framework) {framework.scope(WorkspaceScope).service(DocsSearchService, [WorkspaceService])- .entity(DocsIndexer, [WorkspaceService]);+ .entity(DocsIndexer, [WorkspaceService, WorkspaceLocalState]);}diff --git a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts b/packages/frontend/core/src/modules/docs-search/worker/in-worker.tsindex 40a91304db..640d142fc7 100644--- a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts+++ b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts@@ -285,6 +285,7 @@ async function crawlingDocData({function crawlingRootDocData({allIndexedDocs,rootDocBuffer,+ reindexAll,}: WorkerInput & {type: 'rootDoc';}): WorkerOutput {@@ -317,7 +318,9 @@ function crawlingRootDocData({}const needDelete = difference(allIndexedDocs, availableDocs);- const needAdd = difference(availableDocs, allIndexedDocs);+ const needAdd = reindexAll+ ? availableDocs+ : difference(availableDocs, allIndexedDocs);return {reindexDoc: [...needAdd, ...needDelete].map(docId => ({diff --git a/packages/frontend/core/src/modules/docs-search/worker/types.ts b/packages/frontend/core/src/modules/docs-search/worker/types.tsindex 9240ac63cc..b3d33a8ec7 100644--- a/packages/frontend/core/src/modules/docs-search/worker/types.ts+++ b/packages/frontend/core/src/modules/docs-search/worker/types.ts@@ -31,6 +31,7 @@ export type WorkerInput =type: 'rootDoc';rootDocBuffer: Uint8Array;allIndexedDocs: string[];+ reindexAll?: boolean;}| {type: 'doc';