Skip to content

There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.

feat(nbstore): add indexer sync version (#11324)

EYHNbc0f32f20bef
diff --git a/packages/common/nbstore/src/impls/idb/indexer-sync.ts b/packages/common/nbstore/src/impls/idb/indexer-sync.ts
index 690fa86078..5d0faf4f8b 100644
--- a/packages/common/nbstore/src/impls/idb/indexer-sync.ts
+++ b/packages/common/nbstore/src/impls/idb/indexer-sync.ts
@@ -1,6 +1,8 @@
import { share } from '../../connection';
-import type { DocClock } from '../../storage/doc';
-import { IndexerSyncStorageBase } from '../../storage/indexer-sync';
+import {
+ type DocIndexedClock,
+ IndexerSyncStorageBase,
+} from '../../storage/indexer-sync';
import { IDBConnection, type IDBConnectionOptions } from './db';
export class IndexedDBIndexerSyncStorage extends IndexerSyncStorageBase {
@@ -12,21 +14,26 @@ export class IndexedDBIndexerSyncStorage extends IndexerSyncStorageBase {
super();
}
- async getDocIndexedClock(docId: string): Promise<DocClock | null> {
+ async getDocIndexedClock(docId: string): Promise<DocIndexedClock | null> {
const tx = this.connection.inner.db.transaction('indexerSync', 'readonly');
const store = tx.store;
const result = await store.get(docId);
return result
- ? { docId: result.docId, timestamp: result.indexedClock }
+ ? {
+ docId: result.docId,
+ timestamp: result.indexedClock,
+ indexerVersion: result.indexerVersion ?? 0,
+ }
: null;
}
- async setDocIndexedClock(docClock: DocClock): Promise<void> {
+ async setDocIndexedClock(docClock: DocIndexedClock): Promise<void> {
const tx = this.connection.inner.db.transaction('indexerSync', 'readwrite');
const store = tx.store;
await store.put({
docId: docClock.docId,
indexedClock: docClock.timestamp,
+ indexerVersion: docClock.indexerVersion,
});
}
diff --git a/packages/common/nbstore/src/impls/idb/schema.ts b/packages/common/nbstore/src/impls/idb/schema.ts
index 35e206c2cb..a27cf6fce5 100644
--- a/packages/common/nbstore/src/impls/idb/schema.ts
+++ b/packages/common/nbstore/src/impls/idb/schema.ts
@@ -38,9 +38,9 @@ Table(PeerClocks)
| str | str | Date | Date |
Table(IndexerSync)
-| docId | clock |
-|-------|-------|
-| str | Date |
+| docId | indexedClock | indexerVersion |
+|-------|--------------|----------------|
+| str | Date | number |
Table(BlobSync)
| peer | key | uploadedAt |
@@ -134,6 +134,7 @@ export interface DocStorageSchema extends DBSchema {
value: {
docId: string;
indexedClock: Date;
+ indexerVersion?: number;
};
};
indexerMetadata: {
diff --git a/packages/common/nbstore/src/storage/dummy/indexer-sync.ts b/packages/common/nbstore/src/storage/dummy/indexer-sync.ts
index 96ec17f744..a3adcf73cd 100644
--- a/packages/common/nbstore/src/storage/dummy/indexer-sync.ts
+++ b/packages/common/nbstore/src/storage/dummy/indexer-sync.ts
@@ -1,13 +1,12 @@
import { DummyConnection } from '../../connection';
-import type { DocClock } from '../doc';
-import { IndexerSyncStorageBase } from '../indexer-sync';
+import { type DocIndexedClock, IndexerSyncStorageBase } from '../indexer-sync';
export class DummyIndexerSyncStorage extends IndexerSyncStorageBase {
override connection = new DummyConnection();
- override getDocIndexedClock(_docId: string): Promise<DocClock | null> {
+ override getDocIndexedClock(_docId: string): Promise<DocIndexedClock | null> {
return Promise.resolve(null);
}
- override setDocIndexedClock(_docClock: DocClock): Promise<void> {
+ override setDocIndexedClock(_docClock: DocIndexedClock): Promise<void> {
return Promise.resolve();
}
override clearDocIndexedClock(_docId: string): Promise<void> {
diff --git a/packages/common/nbstore/src/storage/indexer-sync.ts b/packages/common/nbstore/src/storage/indexer-sync.ts
index 8be6c8b5d3..3cb8168294 100644
--- a/packages/common/nbstore/src/storage/indexer-sync.ts
+++ b/packages/common/nbstore/src/storage/indexer-sync.ts
@@ -2,12 +2,16 @@ import type { Connection } from '../connection';
import type { DocClock } from './doc';
import type { Storage } from './storage';
+export interface DocIndexedClock extends DocClock {
+ indexerVersion: number;
+}
+
export interface IndexerSyncStorage extends Storage {
readonly storageType: 'indexerSync';
- getDocIndexedClock(docId: string): Promise<DocClock | null>;
+ getDocIndexedClock(docId: string): Promise<DocIndexedClock | null>;
- setDocIndexedClock(docClock: DocClock): Promise<void>;
+ setDocIndexedClock(docClock: DocIndexedClock): Promise<void>;
clearDocIndexedClock(docId: string): Promise<void>;
}
@@ -15,7 +19,7 @@ export interface IndexerSyncStorage extends Storage {
export abstract class IndexerSyncStorageBase implements IndexerSyncStorage {
readonly storageType = 'indexerSync';
abstract connection: Connection<any>;
- abstract getDocIndexedClock(docId: string): Promise<DocClock | null>;
- abstract setDocIndexedClock(docClock: DocClock): Promise<void>;
+ abstract getDocIndexedClock(docId: string): Promise<DocIndexedClock | null>;
+ abstract setDocIndexedClock(docClock: DocIndexedClock): Promise<void>;
abstract clearDocIndexedClock(docId: string): Promise<void>;
}
diff --git a/packages/common/nbstore/src/sync/indexer/index.ts b/packages/common/nbstore/src/sync/indexer/index.ts
index ff453f54ee..55d6a50b74 100644
--- a/packages/common/nbstore/src/sync/indexer/index.ts
+++ b/packages/common/nbstore/src/sync/indexer/index.ts
@@ -65,6 +65,10 @@ export interface IndexerSync {
}
export class IndexerSyncImpl implements IndexerSync {
+ /**
+ * increase this number to re-index all docs
+ */
+ readonly INDEXER_VERSION = 1;
private abort: AbortController | null = null;
private readonly rootDocId = this.doc.spaceId;
private readonly status = new IndexerSyncStatus(this.rootDocId);
@@ -351,7 +355,9 @@ export class IndexerSyncImpl implements IndexerSync {
await this.indexerSync.getDocIndexedClock(docId);
if (
docIndexedClock &&
- docIndexedClock.timestamp.getTime() === docClock.timestamp.getTime()
+ docIndexedClock.timestamp.getTime() ===
+ docClock.timestamp.getTime() &&
+ docIndexedClock.indexerVersion === this.INDEXER_VERSION
) {
// doc is already indexed, just skip
continue;
@@ -406,6 +412,7 @@ export class IndexerSyncImpl implements IndexerSync {
await this.indexerSync.setDocIndexedClock({
docId,
timestamp: docClock.timestamp,
+ indexerVersion: this.INDEXER_VERSION,
});
// #endregion
}