A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
feat: batch meilisearch requests (#2441)
Mohamed Bassem54243b8cc5cc
diff --git a/apps/workers/workers/searchWorker.ts b/apps/workers/workers/searchWorker.tsindex 7a9c3a8d..b0608dce 100644--- a/apps/workers/workers/searchWorker.ts+++ b/apps/workers/workers/searchWorker.ts@@ -56,7 +56,11 @@ export class SearchIndexingWorker {}}-async function runIndex(searchClient: SearchIndexClient, bookmarkId: string) {+async function runIndex(+ searchClient: SearchIndexClient,+ bookmarkId: string,+ batch: boolean,+) {const bookmark = await db.query.bookmarks.findFirst({where: eq(bookmarks.id, bookmarkId),with: {@@ -107,11 +111,15 @@ async function runIndex(searchClient: SearchIndexClient, bookmarkId: string) {tags: bookmark.tagsOnBookmarks.map((t) => t.tag.name),};- await searchClient.addDocuments([document]);+ await searchClient.addDocuments([document], { batch });}-async function runDelete(searchClient: SearchIndexClient, bookmarkId: string) {- await searchClient.deleteDocuments([bookmarkId]);+async function runDelete(+ searchClient: SearchIndexClient,+ bookmarkId: string,+ batch: boolean,+) {+ await searchClient.deleteDocuments([bookmarkId], { batch });}async function runSearchIndexing(job: DequeuedJob<ZSearchIndexingRequest>) {@@ -133,17 +141,20 @@ async function runSearchIndexing(job: DequeuedJob<ZSearchIndexingRequest>) {}const bookmarkId = request.data.bookmarkId;+ // Disable batching on retries (runNumber > 0) for improved reliability+ const batch = job.runNumber === 0;+logger.info(- `[search][${jobId}] Attempting to index bookmark with id ${bookmarkId} ...`,+ `[search][${jobId}] Attempting to index bookmark with id ${bookmarkId} (run ${job.runNumber}, batch=${batch}) ...`,);switch (request.data.type) {case "index": {- await runIndex(searchClient, bookmarkId);+ await runIndex(searchClient, bookmarkId, batch);break;}case "delete": {- await runDelete(searchClient, bookmarkId);+ await runDelete(searchClient, bookmarkId, batch);break;}}diff --git a/packages/plugins/package.json b/packages/plugins/package.jsonindex daf46843..f263d593 100644--- a/packages/plugins/package.json+++ b/packages/plugins/package.json@@ -22,6 +22,7 @@"@karakeep/shared": "workspace:*","@restatedev/restate-sdk": "^1.9.0","@restatedev/restate-sdk-clients": "^1.9.0",+ "async-mutex": "^0.4.1","liteque": "^0.7.0","meilisearch": "^0.45.0"},diff --git a/packages/plugins/search-meilisearch/src/env.ts b/packages/plugins/search-meilisearch/src/env.tsindex c06fdd55..50800950 100644--- a/packages/plugins/search-meilisearch/src/env.ts+++ b/packages/plugins/search-meilisearch/src/env.ts@@ -4,5 +4,7 @@ export const envConfig = z.object({MEILI_ADDR: z.string().optional(),MEILI_MASTER_KEY: z.string().default(""),+ MEILI_BATCH_SIZE: z.coerce.number().int().positive().default(50),+ MEILI_BATCH_TIMEOUT_MS: z.coerce.number().int().positive().default(500),}).parse(process.env);diff --git a/packages/plugins/search-meilisearch/src/index.ts b/packages/plugins/search-meilisearch/src/index.tsindex 8ffd48cf..5ebbd2ec 100644--- a/packages/plugins/search-meilisearch/src/index.ts+++ b/packages/plugins/search-meilisearch/src/index.ts@@ -1,9 +1,11 @@import type { Index } from "meilisearch";+import { Mutex } from "async-mutex";import { MeiliSearch } from "meilisearch";import type {BookmarkSearchDocument,FilterQuery,+ IndexingOptions,SearchIndexClient,SearchOptions,SearchResponse,@@ -26,19 +28,190 @@ function filterToMeiliSearchFilter(filter: FilterQuery): string {}}-class MeiliSearchIndexClient implements SearchIndexClient {- constructor(private index: Index<BookmarkSearchDocument>) {}+type PendingOperation =+ | {+ type: "add";+ document: BookmarkSearchDocument;+ resolve: () => void;+ reject: (error: Error) => void;+ }+ | {+ type: "delete";+ id: string;+ resolve: () => void;+ reject: (error: Error) => void;+ };++class BatchingDocumentQueue {+ private pendingOperations: PendingOperation[] = [];+ private flushTimeout: ReturnType<typeof setTimeout> | null = null;+ private mutex = new Mutex();- async addDocuments(documents: BookmarkSearchDocument[]): Promise<void> {- const task = await this.index.addDocuments(documents, {- primaryKey: "id",+ constructor(+ private index: Index<BookmarkSearchDocument>,+ private jobTimeoutSec: number,+ private batchSize: number,+ private batchTimeoutMs: number,+ ) {}++ async addDocument(document: BookmarkSearchDocument): Promise<void> {+ return new Promise((resolve, reject) => {+ this.pendingOperations.push({ type: "add", document, resolve, reject });+ this.scheduleFlush();++ if (this.pendingOperations.length >= this.batchSize) {+ void this.flush();+ }});- await this.ensureTaskSuccess(task.taskUid);}- async deleteDocuments(ids: string[]): Promise<void> {- const task = await this.index.deleteDocuments(ids);- await this.ensureTaskSuccess(task.taskUid);+ async deleteDocument(id: string): Promise<void> {+ return new Promise((resolve, reject) => {+ this.pendingOperations.push({ type: "delete", id, resolve, reject });+ this.scheduleFlush();++ if (this.pendingOperations.length >= this.batchSize) {+ void this.flush();+ }+ });+ }++ private scheduleFlush(): void {+ if (this.flushTimeout === null) {+ this.flushTimeout = setTimeout(() => {+ void this.flush();+ }, this.batchTimeoutMs);+ }+ }++ private async flush(): Promise<void> {+ await this.mutex.runExclusive(async () => {+ if (this.flushTimeout) {+ clearTimeout(this.flushTimeout);+ this.flushTimeout = null;+ }++ // Process operations in order, batching consecutive operations of the same type+ while (this.pendingOperations.length > 0) {+ const currentType = this.pendingOperations[0].type;++ // Collect consecutive operations of the same type (up to batchSize)+ const batch: PendingOperation[] = [];+ while (+ batch.length < this.batchSize &&+ this.pendingOperations.length > 0 &&+ this.pendingOperations[0].type === currentType+ ) {+ batch.push(this.pendingOperations.shift()!);+ }++ if (currentType === "add") {+ await this.flushAddBatch(+ batch as Extract<PendingOperation, { type: "add" }>[],+ );+ } else {+ await this.flushDeleteBatch(+ batch as Extract<PendingOperation, { type: "delete" }>[],+ );+ }+ }+ });+ }++ private async flushAddBatch(+ batch: Extract<PendingOperation, { type: "add" }>[],+ ): Promise<void> {+ if (batch.length === 0) return;++ try {+ const documents = batch.map((p) => p.document);+ const task = await this.index.addDocuments(documents, {+ primaryKey: "id",+ });+ await this.ensureTaskSuccess(task.taskUid);+ batch.forEach((p) => p.resolve());+ } catch (error) {+ batch.forEach((p) => p.reject(error as Error));+ }+ }++ private async flushDeleteBatch(+ batch: Extract<PendingOperation, { type: "delete" }>[],+ ): Promise<void> {+ if (batch.length === 0) return;++ try {+ const ids = batch.map((p) => p.id);+ const task = await this.index.deleteDocuments(ids);+ await this.ensureTaskSuccess(task.taskUid);+ batch.forEach((p) => p.resolve());+ } catch (error) {+ batch.forEach((p) => p.reject(error as Error));+ }+ }++ private async ensureTaskSuccess(taskUid: number): Promise<void> {+ const task = await this.index.waitForTask(taskUid, {+ intervalMs: 200,+ timeOutMs: this.jobTimeoutSec * 1000 * 0.9,+ });+ if (task.error) {+ throw new Error(`Search task failed: ${task.error.message}`);+ }+ }+}++class MeiliSearchIndexClient implements SearchIndexClient {+ private batchQueue: BatchingDocumentQueue;+ private jobTimeoutSec: number;++ constructor(+ private index: Index<BookmarkSearchDocument>,+ jobTimeoutSec: number,+ batchSize: number,+ batchTimeoutMs: number,+ ) {+ this.jobTimeoutSec = jobTimeoutSec;+ this.batchQueue = new BatchingDocumentQueue(+ index,+ jobTimeoutSec,+ batchSize,+ batchTimeoutMs,+ );+ }++ async addDocuments(+ documents: BookmarkSearchDocument[],+ options?: IndexingOptions,+ ): Promise<void> {+ const shouldBatch = options?.batch !== false;++ if (shouldBatch) {+ await Promise.all(+ documents.map((doc) => this.batchQueue.addDocument(doc)),+ );+ } else {+ // Direct indexing without batching+ const task = await this.index.addDocuments(documents, {+ primaryKey: "id",+ });+ await this.ensureTaskSuccess(task.taskUid);+ }+ }++ async deleteDocuments(+ ids: string[],+ options?: IndexingOptions,+ ): Promise<void> {+ const shouldBatch = options?.batch !== false;++ if (shouldBatch) {+ await Promise.all(ids.map((id) => this.batchQueue.deleteDocument(id)));+ } else {+ // Direct deletion without batching+ const task = await this.index.deleteDocuments(ids);+ await this.ensureTaskSuccess(task.taskUid);+ }}async search(options: SearchOptions): Promise<SearchResponse> {@@ -69,7 +242,7 @@ class MeiliSearchIndexClient implements SearchIndexClient {private async ensureTaskSuccess(taskUid: number): Promise<void> {const task = await this.index.waitForTask(taskUid, {intervalMs: 200,- timeOutMs: serverConfig.search.jobTimeoutSec * 1000 * 0.9,+ timeOutMs: this.jobTimeoutSec * 1000 * 0.9,});if (task.error) {throw new Error(`Search task failed: ${task.error.message}`);@@ -130,7 +303,12 @@ export class MeiliSearchProvider implements PluginProvider<SearchIndexClient> {}await this.configureIndex(indexFound);- this.indexClient = new MeiliSearchIndexClient(indexFound);+ this.indexClient = new MeiliSearchIndexClient(+ indexFound,+ serverConfig.search.jobTimeoutSec,+ envConfig.MEILI_BATCH_SIZE,+ envConfig.MEILI_BATCH_TIMEOUT_MS,+ );return this.indexClient;}diff --git a/packages/shared/search.ts b/packages/shared/search.tsindex d23ab29f..651b5245 100644--- a/packages/shared/search.ts+++ b/packages/shared/search.ts@@ -60,9 +60,20 @@ export interface SearchResponse {processingTimeMs: number;}+export interface IndexingOptions {+ /**+ * Whether to batch requests. Defaults to true.+ * Set to false to bypass batching for improved reliability (e.g., on retries).+ */+ batch?: boolean;+}+export interface SearchIndexClient {- addDocuments(documents: BookmarkSearchDocument[]): Promise<void>;- deleteDocuments(ids: string[]): Promise<void>;+ addDocuments(+ documents: BookmarkSearchDocument[],+ options?: IndexingOptions,+ ): Promise<void>;+ deleteDocuments(ids: string[], options?: IndexingOptions): Promise<void>;search(options: SearchOptions): Promise<SearchResponse>;clearIndex(): Promise<void>;}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 04361d9c..ba1f7bbd 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1271,6 +1271,9 @@ importers:'@restatedev/restate-sdk-clients':specifier: ^1.9.0version: 1.9.0+ async-mutex:+ specifier: ^0.4.1+ version: 0.4.1liteque:specifier: ^0.7.0version: 0.7.0(@opentelemetry/[email protected])(@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])