A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix: dedup bookmarks during meilisearch batching
Mohamed Bassembc14214bf33e
diff --git a/packages/plugins/search-meilisearch/src/index.ts b/packages/plugins/search-meilisearch/src/index.tsindex 84a20b2c..5ae9f3c3 100644--- a/packages/plugins/search-meilisearch/src/index.ts+++ b/packages/plugins/search-meilisearch/src/index.ts@@ -92,44 +92,73 @@ class BatchingDocumentQueue {this.flushTimeout = null;}- // Process operations in order, batching consecutive operations of the same type- while (this.pendingOperations.length > 0) {- const currentType = this.pendingOperations[0].type;-- // Collect consecutive operations of the same type (up to batchSize)- const batch: PendingOperation[] = [];- while (- batch.length < this.batchSize &&- this.pendingOperations.length > 0 &&- this.pendingOperations[0].type === currentType- ) {- batch.push(this.pendingOperations.shift()!);+ if (this.pendingOperations.length === 0) return;++ // Deduplicate: for each document ID, only the last operation matters.+ // Earlier operations for the same document are resolved immediately since+ // the final state will be achieved by the last operation.+ const lastOpIndexByDocId = new Map<string, number>();+ for (let i = 0; i < this.pendingOperations.length; i++) {+ const op = this.pendingOperations[i];+ const docId = op.type === "add" ? op.document.id : op.id;+ lastOpIndexByDocId.set(docId, i);+ }++ const adds: Extract<PendingOperation, { type: "add" }>[] = [];+ const deletes: Extract<PendingOperation, { type: "delete" }>[] = [];+ const supersededByDocId = new Map<string, PendingOperation[]>();++ for (let i = 0; i < this.pendingOperations.length; i++) {+ const op = this.pendingOperations[i];+ const docId = op.type === "add" ? op.document.id : op.id;++ if (lastOpIndexByDocId.get(docId) !== i) {+ let list = supersededByDocId.get(docId);+ if (!list) {+ list = [];+ supersededByDocId.set(docId, list);+ }+ list.push(op);+ continue;}- let reason: string;- if (batch.length >= this.batchSize) {- reason = "batch size limit reached";- } else if (- this.pendingOperations.length > 0 &&- this.pendingOperations[0].type !== currentType- ) {- reason = "operation type changed";+ // Wrap resolve/reject to also settle any superseded operations+ // for the same document, so callers only see success/failure+ // after the actual batch completes.+ const superseded = supersededByDocId.get(docId) ?? [];+ const origResolve = op.resolve;+ const origReject = op.reject;+ op.resolve = () => {+ origResolve();+ superseded.forEach((s) => s.resolve());+ };+ op.reject = (error: Error) => {+ origReject(error);+ superseded.forEach((s) => s.reject(error));+ };++ if (op.type === "add") {+ adds.push(op as Extract<PendingOperation, { type: "add" }>);} else {- reason = "no more pending operations";+ deletes.push(op as Extract<PendingOperation, { type: "delete" }>);}+ }++ this.pendingOperations = [];++ // Flush all deletes first, then all adds, in batchSize chunks+ for (let i = 0; i < deletes.length; i += this.batchSize) {+ const batch = deletes.slice(i, i + this.batchSize);logger.debug(- `[meilisearch] Flushing ${currentType} batch: size=${batch.length}, remaining=${this.pendingOperations.length}, reason="${reason}"`,+ `[meilisearch] Flushing delete batch: size=${batch.length}`,);+ await this.flushDeleteBatch(batch);+ }- if (currentType === "add") {- await this.flushAddBatch(- batch as Extract<PendingOperation, { type: "add" }>[],- );- } else {- await this.flushDeleteBatch(- batch as Extract<PendingOperation, { type: "delete" }>[],- );- }+ for (let i = 0; i < adds.length; i += this.batchSize) {+ const batch = adds.slice(i, i + this.batchSize);+ logger.debug(`[meilisearch] Flushing add batch: size=${batch.length}`);+ await this.flushAddBatch(batch);}});}