Skip to content

There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.

feat(core): indexer upgrade (#8062)

EYHNfb64bc7e55a8
diff --git a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts
index e506ec2e63..a54e304f31 100644
--- a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts
+++ b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts
@@ -1,5 +1,10 @@
import { DebugLogger } from '@affine/debug';
-import type { Job, JobQueue, WorkspaceService } from '@toeverything/infra';
+import type {
+ Job,
+ JobQueue,
+ WorkspaceLocalState,
+ WorkspaceService,
+} from '@toeverything/infra';
import {
Entity,
IndexedDBIndexStorage,
@@ -21,12 +26,18 @@ export function isEmptyUpdate(binary: Uint8Array) {
}
const logger = new DebugLogger('crawler');
+const WORKSPACE_DOCS_INDEXER_VERSION_KEY = 'docs-indexer-version';
interface IndexerJobPayload {
storageDocId: string;
}
export class DocsIndexer extends Entity {
+ /**
+ * increase this number to re-index all docs
+ */
+ static INDEXER_VERSION = 1;
+
private readonly jobQueue: JobQueue<IndexerJobPayload> =
new IndexedDBJobQueue<IndexerJobPayload>(
'jq:' + this.workspaceService.workspace.id
@@ -66,7 +77,10 @@ export class DocsIndexer extends Entity {
{}
);
- constructor(private readonly workspaceService: WorkspaceService) {
+ constructor(
+ private readonly workspaceService: WorkspaceService,
+ private readonly workspaceLocalState: WorkspaceLocalState
+ ) {
super();
}
@@ -96,6 +110,16 @@ export class DocsIndexer extends Entity {
return;
}
+ const dbVersion = this.getVersion();
+
+ if (dbVersion > DocsIndexer.INDEXER_VERSION) {
+ // stop if db version is higher then self
+ this.runner.stop();
+ throw new Error('Indexer is outdated');
+ }
+
+ const isUpgrade = dbVersion < DocsIndexer.INDEXER_VERSION;
+
// jobs should have the same storage docId, so we just pick the first one
const storageDocId = jobs[0].payload.storageDocId;
@@ -111,7 +135,6 @@ export class DocsIndexer extends Entity {
await this.workspaceEngine.doc.storage.loadDocFromLocal(
this.workspaceId
);
-
if (!rootDocBuffer) {
return;
}
@@ -134,6 +157,7 @@ export class DocsIndexer extends Entity {
type: 'rootDoc',
allIndexedDocs,
rootDocBuffer,
+ reindexAll: isUpgrade,
});
} else {
const rootDocBuffer =
@@ -226,6 +250,10 @@ export class DocsIndexer extends Entity {
);
}
+ if (isUpgrade) {
+ this.setVersion();
+ }
+
const duration = performance.now() - startTime;
logger.debug(
'Finish crawling job for storageDocId:' +
@@ -238,6 +266,7 @@ export class DocsIndexer extends Entity {
startCrawling() {
this.runner.start();
+
this.jobQueue
.enqueue([
{
@@ -257,6 +286,27 @@ export class DocsIndexer extends Entity {
return this.worker;
}
+ getVersion() {
+ const version = this.workspaceLocalState.get<number>(
+ WORKSPACE_DOCS_INDEXER_VERSION_KEY
+ );
+ if (typeof version !== 'number') {
+ return -1;
+ } else {
+ return version;
+ }
+ }
+
+ setVersion(version = DocsIndexer.INDEXER_VERSION) {
+ if (this.getVersion() >= version) {
+ return;
+ }
+ return this.workspaceLocalState.set(
+ WORKSPACE_DOCS_INDEXER_VERSION_KEY,
+ version
+ );
+ }
+
override dispose(): void {
this.runner.stop();
}
diff --git a/packages/frontend/core/src/modules/docs-search/index.ts b/packages/frontend/core/src/modules/docs-search/index.ts
index 19df647ba9..03fd0e157a 100644
--- a/packages/frontend/core/src/modules/docs-search/index.ts
+++ b/packages/frontend/core/src/modules/docs-search/index.ts
@@ -2,6 +2,7 @@ export { DocsSearchService } from './services/docs-search';
import {
type Framework,
+ WorkspaceLocalState,
WorkspaceScope,
WorkspaceService,
} from '@toeverything/infra';
@@ -13,5 +14,5 @@ export function configureDocsSearchModule(framework: Framework) {
framework
.scope(WorkspaceScope)
.service(DocsSearchService, [WorkspaceService])
- .entity(DocsIndexer, [WorkspaceService]);
+ .entity(DocsIndexer, [WorkspaceService, WorkspaceLocalState]);
}
diff --git a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts
index 40a91304db..640d142fc7 100644
--- a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts
+++ b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts
@@ -285,6 +285,7 @@ async function crawlingDocData({
function crawlingRootDocData({
allIndexedDocs,
rootDocBuffer,
+ reindexAll,
}: WorkerInput & {
type: 'rootDoc';
}): WorkerOutput {
@@ -317,7 +318,9 @@ function crawlingRootDocData({
}
const needDelete = difference(allIndexedDocs, availableDocs);
- const needAdd = difference(availableDocs, allIndexedDocs);
+ const needAdd = reindexAll
+ ? availableDocs
+ : difference(availableDocs, allIndexedDocs);
return {
reindexDoc: [...needAdd, ...needDelete].map(docId => ({
diff --git a/packages/frontend/core/src/modules/docs-search/worker/types.ts b/packages/frontend/core/src/modules/docs-search/worker/types.ts
index 9240ac63cc..b3d33a8ec7 100644
--- a/packages/frontend/core/src/modules/docs-search/worker/types.ts
+++ b/packages/frontend/core/src/modules/docs-search/worker/types.ts
@@ -31,6 +31,7 @@ export type WorkerInput =
type: 'rootDoc';
rootDocBuffer: Uint8Array;
allIndexedDocs: string[];
+ reindexAll?: boolean;
}
| {
type: 'doc';