A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix: fast track tagging after embedding generation (#2860)
Mohamed Bassem63694167b680
diff --git a/apps/workers/workers/assetPreprocessingWorker.ts b/apps/workers/workers/assetPreprocessingWorker.tsindex 5b36f23c..adbb1a2b 100644--- a/apps/workers/workers/assetPreprocessingWorker.ts+++ b/apps/workers/workers/assetPreprocessingWorker.ts@@ -477,7 +477,8 @@ async function run(req: DequeuedJob<AssetPreprocessingRequest>) {await EmbeddingsQueue.enqueue({bookmarkId,- type: "index",+ type: "embed",+ runTaggingOnComplete: true,},enqueueOpts,);diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.tsindex 0a0cb5d9..cef6ea28 100644--- a/apps/workers/workers/crawlerWorker.ts+++ b/apps/workers/workers/crawlerWorker.ts@@ -2313,7 +2313,8 @@ async function runCrawler(await EmbeddingsQueue.enqueue({bookmarkId,- type: "index",+ type: "embed",+ runTaggingOnComplete: true,},enqueueOpts,);diff --git a/apps/workers/workers/embeddingsWorker.ts b/apps/workers/workers/embeddingsWorker.tsindex 9a5bac40..8899d90d 100644--- a/apps/workers/workers/embeddingsWorker.ts+++ b/apps/workers/workers/embeddingsWorker.ts@@ -17,7 +17,6 @@ import logger from "@karakeep/shared/logger";import {DequeuedJob,DequeuedJobError,- EnqueueOptions,getQueueClient,} from "@karakeep/shared/queueing";import {@@ -44,9 +43,6 @@ export class EmbeddingsWorker {const jobId = job.id;logger.info(`[embeddings][${jobId}] Completed successfully`);await attemptMarkEmbeddingStatus(job.data, "success");- if (shouldRunTaggingOnComplete(job.data)) {- await enqueueTaggingAfterEmbeddings(job);- }},onError: async (job) => {workerStatsCounter.labels("embeddings", "failed").inc();@@ -57,8 +53,13 @@ export class EmbeddingsWorker {if (job.numRetriesLeft == 0) {workerStatsCounter.labels("embeddings", "failed_permanent").inc();await attemptMarkEmbeddingStatus(job.data, "failure");- if (job.data && shouldRunTaggingOnComplete(job.data)) {- await enqueueTaggingAfterEmbeddings(job);+ // If embedding generation permanently failed, still tag the bookmark+ // (without similarity context) so it isn't left untagged.+ if (+ job.data?.type === "embed" &&+ job.data.runTaggingOnComplete !== false+ ) {+ await enqueueTaggingFallback(job);}}},@@ -84,7 +85,10 @@ async function attemptMarkEmbeddingStatus(}try {const request = zEmbeddingsRequestSchema.parse(jobData);- if (request.type !== "index") {+ if (+ request.type !== "index" &&+ !(request.type === "embed" && status === "failure")+ ) {return;}await db@@ -98,11 +102,29 @@ async function attemptMarkEmbeddingStatus(}}-function shouldRunTaggingOnComplete(request: ZEmbeddingsRequest) {- return request.type !== "delete" && request.runTaggingOnComplete !== false;+async function enqueueTagging(+ bookmarkId: string,+ userId: string,+ priority: number | undefined,+ embedding?: number[],+) {+ await OpenAIQueue.enqueue(+ {+ bookmarkId,+ type: "tag",+ ...(embedding ? { embedding } : {}),+ },+ {+ priority,+ groupId: userId,+ },+ );}-async function enqueueTaggingAfterEmbeddings(+// Enqueue a vector-less tag job when the embed run produced no embedding (early+// returns / permanent generation failure). Looks up the user since the embed+// payload only carries the bookmark id.+async function enqueueTaggingFallback(job: DequeuedJob<ZEmbeddingsRequest> | DequeuedJobError<ZEmbeddingsRequest>,) {const bookmarkId = job.data?.bookmarkId;@@ -121,18 +143,7 @@ async function enqueueTaggingAfterEmbeddings();return;}-- const enqueueOpts: EnqueueOptions = {- priority: job.priority,- groupId: bookmark.userId,- };- await OpenAIQueue.enqueue(- {- bookmarkId,- type: "tag",- },- enqueueOpts,- );+ await enqueueTagging(bookmarkId, bookmark.userId, job.priority);}async function fetchBookmark(bookmarkId: string) {@@ -332,36 +343,73 @@ async function buildEmbeddingText(: fullText;}+type EmbedRequest = Extract<ZEmbeddingsRequest, { type: "embed" }>;+async function runEmbeddings(job: DequeuedJob<ZEmbeddingsRequest>) {const jobId = job.id;-- const { bookmarkId, force, type: opType } = job.data;+ const data = job.data;+ const bookmarkId = data.bookmarkId;addLogFields<"embeddingsWorker.run">({"bookmark.id": bookmarkId,+ "embedding.mode": data.type,});- if (- opType === "index" &&- !serverConfig.embedding.enableAutoIndexing &&- !force- ) {- logger.debug(- `[embeddings][${jobId}] Bookmark embedding indexing is disabled, skipping embedding generation`,- );- return;- }-const vectorStoreClient = await getVectorStoreClient();if (!vectorStoreClient) {logger.debug(- `[embeddings][${jobId}] Vector store is not configured, skipping embedding ${opType}`,+ `[embeddings][${jobId}] Vector store is not configured, skipping embedding ${data.type}`,);+ // The vector cannot be stored, but the bookmark should still be tagged.+ if (data.type === "embed" && data.runTaggingOnComplete !== false) {+ await enqueueTaggingFallback(job);+ }return;}- if (opType === "delete") {- await vectorStoreClient.deleteVectors([bookmarkId]);- logger.info(`[embeddings][${jobId}] Deleted embedding for ${bookmarkId}`);+ switch (data.type) {+ case "delete": {+ await vectorStoreClient.deleteVectors([bookmarkId]);+ logger.info(`[embeddings][${jobId}] Deleted embedding for ${bookmarkId}`);+ return;+ }+ case "index": {+ const document: BookmarkVectorDocument = {+ id: bookmarkId,+ userId: data.userId,+ vector: data.embedding,+ };+ await vectorStoreClient.addVectors([document]);+ logger.info(+ `[embeddings][${jobId}] Indexed embedding for bookmark ${bookmarkId} with ${data.embedding.length} dimensions`,+ );+ return;+ }+ case "embed": {+ await runEmbed(job, data);+ return;+ }+ }+}++// Generates the bookmark embedding and dispatches both the tagging job (carrying+// the vector, so tagging need not wait for indexing) and a separate "index" job+// that persists the vector. This run never calls addVectors, so it does not+// retry on indexing failures and therefore never re-triggers tagging.+async function runEmbed(+ job: DequeuedJob<ZEmbeddingsRequest>,+ data: EmbedRequest,+) {+ const jobId = job.id;+ const { bookmarkId, force } = data;+ const shouldTag = data.runTaggingOnComplete !== false;++ if (!serverConfig.embedding.enableAutoIndexing && !force) {+ logger.debug(+ `[embeddings][${jobId}] Bookmark embedding indexing is disabled, skipping embedding generation`,+ );+ if (shouldTag) {+ await enqueueTaggingFallback(job);+ }return;}@@ -382,6 +430,9 @@ async function runEmbeddings(job: DequeuedJob<ZEmbeddingsRequest>) {logger.debug(`[embeddings][${jobId}] No inference client configured, skipping embedding generation`,);+ if (shouldTag) {+ await enqueueTagging(bookmarkId, bookmark.userId, job.priority);+ }return;}@@ -390,6 +441,9 @@ async function runEmbeddings(job: DequeuedJob<ZEmbeddingsRequest>) {logger.info(`[embeddings][${jobId}] No content found for bookmark ${bookmarkId}, skipping embedding generation`,);+ if (shouldTag) {+ await enqueueTagging(bookmarkId, bookmark.userId, job.priority);+ }return;}@@ -418,15 +472,27 @@ async function runEmbeddings(job: DequeuedJob<ZEmbeddingsRequest>) {"embedding.prompt_tokens": embeddingResponse.promptTokens,"embedding.total_tokens": embeddingResponse.totalTokens,});- const document: BookmarkVectorDocument = {- id: bookmark.id,- userId: bookmark.userId,- vector: embedding,- };- await vectorStoreClient.addVectors([document]);+ // Hand off persistence to a separate "index" job whose retries are isolated+ // from tagging, then dispatch tagging last so nothing after it can throw and+ // cause this run to retry (and double-enqueue tagging).+ await EmbeddingsQueue.enqueue(+ {+ type: "index",+ bookmarkId,+ userId: bookmark.userId,+ embedding,+ },+ {+ priority: job.priority,+ groupId: bookmark.userId,+ },+ );+ if (shouldTag) {+ await enqueueTagging(bookmarkId, bookmark.userId, job.priority, embedding);+ }logger.info(- `[embeddings][${jobId}] Indexed embedding for bookmark ${bookmarkId} with ${embedding.length} dimensions using ${embeddingResponse.totalTokens ?? "unknown"} tokens`,+ `[embeddings][${jobId}] Generated embedding for bookmark ${bookmarkId} with ${embedding.length} dimensions using ${embeddingResponse.totalTokens ?? "unknown"} tokens; dispatched indexing${shouldTag ? " + tagging" : ""}`,);}diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.tsindex 06ccbee7..45b76942 100644--- a/apps/workers/workers/inference/tagging.ts+++ b/apps/workers/workers/inference/tagging.ts@@ -523,31 +523,61 @@ async function fetchBookmark(linkId: string) {});}+// Matches the rankingScoreThreshold the vector store applies in findSimilar, so+// the search({vector}) path returns comparably relevant neighbors.+const RELEVANT_TAG_SCORE_THRESHOLD = 0.75;+/**- * Finds potentially relevant tags for the passed bookmarkId by find similar bookmarks and fetching their tags.+ * Finds potentially relevant tags for the passed bookmarkId by finding similar+ * bookmarks and fetching their tags.+ *+ * When a freshly generated `embedding` is supplied, similarity is resolved via+ * search({vector}) — which does not require the bookmark to be indexed yet — so+ * tagging does not have to wait for the (slow) vector index build. Otherwise it+ * falls back to findSimilar({id}), which requires the bookmark to already be+ * indexed (e.g. a manual re-tag).*/async function getPotentiallyRelevantTags(jobId: string,bookmarkId: string,userId: string,+ embedding?: number[],): Promise<string[] | null> {const client = await getVectorStoreClient();if (!client) {return null;}- const similarBookmarkIds = await client- .findSimilar({- id: bookmarkId,- limit: 10,- filter: [- {- type: "eq",- field: "userId",- value: userId,- },- ],- })- .then((r) => r.hits.map((r) => r.id));+ const userFilter = [+ {+ type: "eq" as const,+ field: "userId" as const,+ value: userId,+ },+ ];+ const similarBookmarkIds =+ embedding && embedding.length > 0+ ? await client+ .search({+ vector: embedding,+ // Fetch one extra so we can drop the bookmark itself if it happens+ // to already be indexed, and still keep up to 10 neighbors.+ limit: 11,+ filter: userFilter,+ rankingScoreThreshold: RELEVANT_TAG_SCORE_THRESHOLD,+ })+ .then((r) =>+ r.hits+ .filter((h) => h.id !== bookmarkId)+ .map((h) => h.id)+ .slice(0, 10),+ )+ : await client+ .findSimilar({+ id: bookmarkId,+ limit: 10,+ filter: userFilter,+ })+ .then((r) => r.hits.map((r) => r.id));if (similarBookmarkIds.length === 0) {return null;@@ -643,6 +673,7 @@ export async function runTagging(jobId,bookmarkId,bookmark.userId,+ job.data.embedding,)) ?? undefined;} catch (e) {logger.error(diff --git a/packages/plugins/vectorstore-meilisearch/src/env.ts b/packages/plugins/vectorstore-meilisearch/src/env.tsindex 50800950..dc8215fd 100644--- a/packages/plugins/vectorstore-meilisearch/src/env.ts+++ b/packages/plugins/vectorstore-meilisearch/src/env.ts@@ -1,10 +1,22 @@import { z } from "zod";-export const envConfig = z+const rawConfig = z.object({MEILI_ADDR: z.string().optional(),- MEILI_MASTER_KEY: z.string().default(""),+ MEILI_MASTER_KEY: z.string().optional(),+ // Dedicated Meilisearch instance for the vector store. When unset, the+ // vector store falls back to the main (search) Meilisearch instance.+ MEILI_VECTOR_ADDR: z.string().optional(),+ MEILI_VECTOR_MASTER_KEY: z.string().optional(),MEILI_BATCH_SIZE: z.coerce.number().int().positive().default(50),MEILI_BATCH_TIMEOUT_MS: z.coerce.number().int().positive().default(500),}).parse(process.env);++export const envConfig = {+ MEILI_ADDR: rawConfig.MEILI_VECTOR_ADDR ?? rawConfig.MEILI_ADDR,+ MEILI_MASTER_KEY:+ rawConfig.MEILI_VECTOR_MASTER_KEY ?? rawConfig.MEILI_MASTER_KEY ?? "",+ MEILI_BATCH_SIZE: rawConfig.MEILI_BATCH_SIZE,+ MEILI_BATCH_TIMEOUT_MS: rawConfig.MEILI_BATCH_TIMEOUT_MS,+};diff --git a/packages/plugins/vectorstore-meilisearch/src/index.ts b/packages/plugins/vectorstore-meilisearch/src/index.tsindex 96f9219e..68602c51 100644--- a/packages/plugins/vectorstore-meilisearch/src/index.ts+++ b/packages/plugins/vectorstore-meilisearch/src/index.ts@@ -87,6 +87,7 @@ class MeiliSearchVectorClient implements VectorStoreClient {limit: options.limit ?? 10,attributesToRetrieve: ["id"],showRankingScore: true,+ rankingScoreThreshold: options.rankingScoreThreshold,});return {diff --git a/packages/shared-server/src/eventLogTypes.ts b/packages/shared-server/src/eventLogTypes.tsindex 40255ea3..1fd10a7a 100644--- a/packages/shared-server/src/eventLogTypes.ts+++ b/packages/shared-server/src/eventLogTypes.ts@@ -81,6 +81,7 @@ type EventLogInternal =| {["event.name"]: "embeddingsWorker.run";"bookmark.id"?: string;+ "embedding.mode"?: "embed" | "index" | "delete";"embedding.text_size"?: number;"embedding.prompt_tokens"?: number;"embedding.total_tokens"?: number;diff --git a/packages/shared-server/src/queues.ts b/packages/shared-server/src/queues.tsindex c0456836..84882bb4 100644--- a/packages/shared-server/src/queues.ts+++ b/packages/shared-server/src/queues.ts@@ -120,6 +120,10 @@ export function buildCrawlIdempotencyKey(payload: ZCrawlLinkRequest): string {export const zOpenAIRequestSchema = z.object({bookmarkId: z.string(),type: z.enum(["summarize", "tag"]).default("tag"),+ // Precomputed embedding so tagging can find similar bookmarks via+ // search({vector}) without waiting for the vector to be indexed. Only set on+ // the embed -> tag path.+ embedding: z.array(z.number()).optional(),});export type ZOpenAIRequest = z.infer<typeof zOpenAIRequestSchema>;@@ -131,12 +135,31 @@ export const OpenAIQueue = createDeferredQueue<ZOpenAIRequest>("openai_queue", {});// Embeddings Worker-export const zEmbeddingsRequestSchema = z.object({- bookmarkId: z.string(),- type: z.enum(["index", "delete"]).default("index"),- force: z.boolean().optional(),- runTaggingOnComplete: z.boolean().optional(),-});+//+// - "embed": entry point. Generates the bookmark embedding, then dispatches the+// tagging job (carrying the vector) and an "index" job. Does NOT persist the+// vector itself, so it never retries on indexing failures.+// - "index": persists a precomputed vector in the vector store. Its retries (the+// slow Meilisearch index build) are isolated and never re-trigger tagging.+// - "delete": removes the vector from the store.+export const zEmbeddingsRequestSchema = z.discriminatedUnion("type", [+ z.object({+ type: z.literal("embed"),+ bookmarkId: z.string(),+ force: z.boolean().optional(),+ runTaggingOnComplete: z.boolean().optional().default(true),+ }),+ z.object({+ type: z.literal("index"),+ bookmarkId: z.string(),+ userId: z.string(),+ embedding: z.array(z.number()),+ }),+ z.object({+ type: z.literal("delete"),+ bookmarkId: z.string(),+ }),+]);export type ZEmbeddingsRequest = z.infer<typeof zEmbeddingsRequestSchema>;export const EmbeddingsQueue = createDeferredQueue<ZEmbeddingsRequest>(diff --git a/packages/shared/vectorStore.ts b/packages/shared/vectorStore.tsindex 314af829..e7dbcee4 100644--- a/packages/shared/vectorStore.ts+++ b/packages/shared/vectorStore.ts@@ -33,6 +33,8 @@ export interface VectorSearchOptions {// Different filters are ANDed togetherfilter?: VectorFilterQuery[];limit?: number;+ // Drop hits whose ranking score is below this threshold (0..1)+ rankingScoreThreshold?: number;}export interface VectorSimilarSearchOptions {diff --git a/packages/trpc/routers/admin.ts b/packages/trpc/routers/admin.tsindex 8cf1cef5..07adc767 100644--- a/packages/trpc/routers/admin.ts+++ b/packages/trpc/routers/admin.ts@@ -343,7 +343,7 @@ export const adminAppRouter = router({EmbeddingsQueue.enqueue({bookmarkId: id,- type: "index",+ type: "embed",force: true,runTaggingOnComplete: false,},@@ -856,7 +856,7 @@ export const adminAppRouter = router({await EmbeddingsQueue.enqueue({bookmarkId: input.bookmarkId,- type: "index",+ type: "embed",force: true,runTaggingOnComplete: false,},diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.tsindex 24f78f85..ea628489 100644--- a/packages/trpc/routers/bookmarks.ts+++ b/packages/trpc/routers/bookmarks.ts@@ -412,7 +412,8 @@ export const bookmarksAppRouter = router({await EmbeddingsQueue.enqueue({bookmarkId: bookmark.id,- type: "index",+ type: "embed",+ runTaggingOnComplete: true,},enqueueOpts,);