Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

fix: fast track tagging after embedding generation (#2860)

Mohamed Bassem63694167b680
diff --git a/apps/workers/workers/assetPreprocessingWorker.ts b/apps/workers/workers/assetPreprocessingWorker.ts
index 5b36f23c..adbb1a2b 100644
--- a/apps/workers/workers/assetPreprocessingWorker.ts
+++ b/apps/workers/workers/assetPreprocessingWorker.ts
@@ -477,7 +477,8 @@ async function run(req: DequeuedJob<AssetPreprocessingRequest>) {
await EmbeddingsQueue.enqueue(
{
bookmarkId,
- type: "index",
+ type: "embed",
+ runTaggingOnComplete: true,
},
enqueueOpts,
);
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index 0a0cb5d9..cef6ea28 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -2313,7 +2313,8 @@ async function runCrawler(
await EmbeddingsQueue.enqueue(
{
bookmarkId,
- type: "index",
+ type: "embed",
+ runTaggingOnComplete: true,
},
enqueueOpts,
);
diff --git a/apps/workers/workers/embeddingsWorker.ts b/apps/workers/workers/embeddingsWorker.ts
index 9a5bac40..8899d90d 100644
--- a/apps/workers/workers/embeddingsWorker.ts
+++ b/apps/workers/workers/embeddingsWorker.ts
@@ -17,7 +17,6 @@ import logger from "@karakeep/shared/logger";
import {
DequeuedJob,
DequeuedJobError,
- EnqueueOptions,
getQueueClient,
} from "@karakeep/shared/queueing";
import {
@@ -44,9 +43,6 @@ export class EmbeddingsWorker {
const jobId = job.id;
logger.info(`[embeddings][${jobId}] Completed successfully`);
await attemptMarkEmbeddingStatus(job.data, "success");
- if (shouldRunTaggingOnComplete(job.data)) {
- await enqueueTaggingAfterEmbeddings(job);
- }
},
onError: async (job) => {
workerStatsCounter.labels("embeddings", "failed").inc();
@@ -57,8 +53,13 @@ export class EmbeddingsWorker {
if (job.numRetriesLeft == 0) {
workerStatsCounter.labels("embeddings", "failed_permanent").inc();
await attemptMarkEmbeddingStatus(job.data, "failure");
- if (job.data && shouldRunTaggingOnComplete(job.data)) {
- await enqueueTaggingAfterEmbeddings(job);
+ // If embedding generation permanently failed, still tag the bookmark
+ // (without similarity context) so it isn't left untagged.
+ if (
+ job.data?.type === "embed" &&
+ job.data.runTaggingOnComplete !== false
+ ) {
+ await enqueueTaggingFallback(job);
}
}
},
@@ -84,7 +85,10 @@ async function attemptMarkEmbeddingStatus(
}
try {
const request = zEmbeddingsRequestSchema.parse(jobData);
- if (request.type !== "index") {
+ if (
+ request.type !== "index" &&
+ !(request.type === "embed" && status === "failure")
+ ) {
return;
}
await db
@@ -98,11 +102,29 @@ async function attemptMarkEmbeddingStatus(
}
}
-function shouldRunTaggingOnComplete(request: ZEmbeddingsRequest) {
- return request.type !== "delete" && request.runTaggingOnComplete !== false;
+async function enqueueTagging(
+ bookmarkId: string,
+ userId: string,
+ priority: number | undefined,
+ embedding?: number[],
+) {
+ await OpenAIQueue.enqueue(
+ {
+ bookmarkId,
+ type: "tag",
+ ...(embedding ? { embedding } : {}),
+ },
+ {
+ priority,
+ groupId: userId,
+ },
+ );
}
-async function enqueueTaggingAfterEmbeddings(
+// Enqueue a vector-less tag job when the embed run produced no embedding (early
+// returns / permanent generation failure). Looks up the user since the embed
+// payload only carries the bookmark id.
+async function enqueueTaggingFallback(
job: DequeuedJob<ZEmbeddingsRequest> | DequeuedJobError<ZEmbeddingsRequest>,
) {
const bookmarkId = job.data?.bookmarkId;
@@ -121,18 +143,7 @@ async function enqueueTaggingAfterEmbeddings(
);
return;
}
-
- const enqueueOpts: EnqueueOptions = {
- priority: job.priority,
- groupId: bookmark.userId,
- };
- await OpenAIQueue.enqueue(
- {
- bookmarkId,
- type: "tag",
- },
- enqueueOpts,
- );
+ await enqueueTagging(bookmarkId, bookmark.userId, job.priority);
}
async function fetchBookmark(bookmarkId: string) {
@@ -332,36 +343,73 @@ async function buildEmbeddingText(
: fullText;
}
+type EmbedRequest = Extract<ZEmbeddingsRequest, { type: "embed" }>;
+
async function runEmbeddings(job: DequeuedJob<ZEmbeddingsRequest>) {
const jobId = job.id;
-
- const { bookmarkId, force, type: opType } = job.data;
+ const data = job.data;
+ const bookmarkId = data.bookmarkId;
addLogFields<"embeddingsWorker.run">({
"bookmark.id": bookmarkId,
+ "embedding.mode": data.type,
});
- if (
- opType === "index" &&
- !serverConfig.embedding.enableAutoIndexing &&
- !force
- ) {
- logger.debug(
- `[embeddings][${jobId}] Bookmark embedding indexing is disabled, skipping embedding generation`,
- );
- return;
- }
-
const vectorStoreClient = await getVectorStoreClient();
if (!vectorStoreClient) {
logger.debug(
- `[embeddings][${jobId}] Vector store is not configured, skipping embedding ${opType}`,
+ `[embeddings][${jobId}] Vector store is not configured, skipping embedding ${data.type}`,
);
+ // The vector cannot be stored, but the bookmark should still be tagged.
+ if (data.type === "embed" && data.runTaggingOnComplete !== false) {
+ await enqueueTaggingFallback(job);
+ }
return;
}
- if (opType === "delete") {
- await vectorStoreClient.deleteVectors([bookmarkId]);
- logger.info(`[embeddings][${jobId}] Deleted embedding for ${bookmarkId}`);
+ switch (data.type) {
+ case "delete": {
+ await vectorStoreClient.deleteVectors([bookmarkId]);
+ logger.info(`[embeddings][${jobId}] Deleted embedding for ${bookmarkId}`);
+ return;
+ }
+ case "index": {
+ const document: BookmarkVectorDocument = {
+ id: bookmarkId,
+ userId: data.userId,
+ vector: data.embedding,
+ };
+ await vectorStoreClient.addVectors([document]);
+ logger.info(
+ `[embeddings][${jobId}] Indexed embedding for bookmark ${bookmarkId} with ${data.embedding.length} dimensions`,
+ );
+ return;
+ }
+ case "embed": {
+ await runEmbed(job, data);
+ return;
+ }
+ }
+}
+
+// Generates the bookmark embedding and dispatches both the tagging job (carrying
+// the vector, so tagging need not wait for indexing) and a separate "index" job
+// that persists the vector. This run never calls addVectors, so it does not
+// retry on indexing failures and therefore never re-triggers tagging.
+async function runEmbed(
+ job: DequeuedJob<ZEmbeddingsRequest>,
+ data: EmbedRequest,
+) {
+ const jobId = job.id;
+ const { bookmarkId, force } = data;
+ const shouldTag = data.runTaggingOnComplete !== false;
+
+ if (!serverConfig.embedding.enableAutoIndexing && !force) {
+ logger.debug(
+ `[embeddings][${jobId}] Bookmark embedding indexing is disabled, skipping embedding generation`,
+ );
+ if (shouldTag) {
+ await enqueueTaggingFallback(job);
+ }
return;
}
@@ -382,6 +430,9 @@ async function runEmbeddings(job: DequeuedJob<ZEmbeddingsRequest>) {
logger.debug(
`[embeddings][${jobId}] No inference client configured, skipping embedding generation`,
);
+ if (shouldTag) {
+ await enqueueTagging(bookmarkId, bookmark.userId, job.priority);
+ }
return;
}
@@ -390,6 +441,9 @@ async function runEmbeddings(job: DequeuedJob<ZEmbeddingsRequest>) {
logger.info(
`[embeddings][${jobId}] No content found for bookmark ${bookmarkId}, skipping embedding generation`,
);
+ if (shouldTag) {
+ await enqueueTagging(bookmarkId, bookmark.userId, job.priority);
+ }
return;
}
@@ -418,15 +472,27 @@ async function runEmbeddings(job: DequeuedJob<ZEmbeddingsRequest>) {
"embedding.prompt_tokens": embeddingResponse.promptTokens,
"embedding.total_tokens": embeddingResponse.totalTokens,
});
- const document: BookmarkVectorDocument = {
- id: bookmark.id,
- userId: bookmark.userId,
- vector: embedding,
- };
- await vectorStoreClient.addVectors([document]);
+ // Hand off persistence to a separate "index" job whose retries are isolated
+ // from tagging, then dispatch tagging last so nothing after it can throw and
+ // cause this run to retry (and double-enqueue tagging).
+ await EmbeddingsQueue.enqueue(
+ {
+ type: "index",
+ bookmarkId,
+ userId: bookmark.userId,
+ embedding,
+ },
+ {
+ priority: job.priority,
+ groupId: bookmark.userId,
+ },
+ );
+ if (shouldTag) {
+ await enqueueTagging(bookmarkId, bookmark.userId, job.priority, embedding);
+ }
logger.info(
- `[embeddings][${jobId}] Indexed embedding for bookmark ${bookmarkId} with ${embedding.length} dimensions using ${embeddingResponse.totalTokens ?? "unknown"} tokens`,
+ `[embeddings][${jobId}] Generated embedding for bookmark ${bookmarkId} with ${embedding.length} dimensions using ${embeddingResponse.totalTokens ?? "unknown"} tokens; dispatched indexing${shouldTag ? " + tagging" : ""}`,
);
}
diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.ts
index 06ccbee7..45b76942 100644
--- a/apps/workers/workers/inference/tagging.ts
+++ b/apps/workers/workers/inference/tagging.ts
@@ -523,31 +523,61 @@ async function fetchBookmark(linkId: string) {
});
}
+// Matches the rankingScoreThreshold the vector store applies in findSimilar, so
+// the search({vector}) path returns comparably relevant neighbors.
+const RELEVANT_TAG_SCORE_THRESHOLD = 0.75;
+
/**
- * Finds potentially relevant tags for the passed bookmarkId by find similar bookmarks and fetching their tags.
+ * Finds potentially relevant tags for the passed bookmarkId by finding similar
+ * bookmarks and fetching their tags.
+ *
+ * When a freshly generated `embedding` is supplied, similarity is resolved via
+ * search({vector}) — which does not require the bookmark to be indexed yet — so
+ * tagging does not have to wait for the (slow) vector index build. Otherwise it
+ * falls back to findSimilar({id}), which requires the bookmark to already be
+ * indexed (e.g. a manual re-tag).
*/
async function getPotentiallyRelevantTags(
jobId: string,
bookmarkId: string,
userId: string,
+ embedding?: number[],
): Promise<string[] | null> {
const client = await getVectorStoreClient();
if (!client) {
return null;
}
- const similarBookmarkIds = await client
- .findSimilar({
- id: bookmarkId,
- limit: 10,
- filter: [
- {
- type: "eq",
- field: "userId",
- value: userId,
- },
- ],
- })
- .then((r) => r.hits.map((r) => r.id));
+ const userFilter = [
+ {
+ type: "eq" as const,
+ field: "userId" as const,
+ value: userId,
+ },
+ ];
+ const similarBookmarkIds =
+ embedding && embedding.length > 0
+ ? await client
+ .search({
+ vector: embedding,
+ // Fetch one extra so we can drop the bookmark itself if it happens
+ // to already be indexed, and still keep up to 10 neighbors.
+ limit: 11,
+ filter: userFilter,
+ rankingScoreThreshold: RELEVANT_TAG_SCORE_THRESHOLD,
+ })
+ .then((r) =>
+ r.hits
+ .filter((h) => h.id !== bookmarkId)
+ .map((h) => h.id)
+ .slice(0, 10),
+ )
+ : await client
+ .findSimilar({
+ id: bookmarkId,
+ limit: 10,
+ filter: userFilter,
+ })
+ .then((r) => r.hits.map((r) => r.id));
if (similarBookmarkIds.length === 0) {
return null;
@@ -643,6 +673,7 @@ export async function runTagging(
jobId,
bookmarkId,
bookmark.userId,
+ job.data.embedding,
)) ?? undefined;
} catch (e) {
logger.error(
diff --git a/packages/plugins/vectorstore-meilisearch/src/env.ts b/packages/plugins/vectorstore-meilisearch/src/env.ts
index 50800950..dc8215fd 100644
--- a/packages/plugins/vectorstore-meilisearch/src/env.ts
+++ b/packages/plugins/vectorstore-meilisearch/src/env.ts
@@ -1,10 +1,22 @@
import { z } from "zod";
-export const envConfig = z
+const rawConfig = z
.object({
MEILI_ADDR: z.string().optional(),
- MEILI_MASTER_KEY: z.string().default(""),
+ MEILI_MASTER_KEY: z.string().optional(),
+ // Dedicated Meilisearch instance for the vector store. When unset, the
+ // vector store falls back to the main (search) Meilisearch instance.
+ MEILI_VECTOR_ADDR: z.string().optional(),
+ MEILI_VECTOR_MASTER_KEY: z.string().optional(),
MEILI_BATCH_SIZE: z.coerce.number().int().positive().default(50),
MEILI_BATCH_TIMEOUT_MS: z.coerce.number().int().positive().default(500),
})
.parse(process.env);
+
+export const envConfig = {
+ MEILI_ADDR: rawConfig.MEILI_VECTOR_ADDR ?? rawConfig.MEILI_ADDR,
+ MEILI_MASTER_KEY:
+ rawConfig.MEILI_VECTOR_MASTER_KEY ?? rawConfig.MEILI_MASTER_KEY ?? "",
+ MEILI_BATCH_SIZE: rawConfig.MEILI_BATCH_SIZE,
+ MEILI_BATCH_TIMEOUT_MS: rawConfig.MEILI_BATCH_TIMEOUT_MS,
+};
diff --git a/packages/plugins/vectorstore-meilisearch/src/index.ts b/packages/plugins/vectorstore-meilisearch/src/index.ts
index 96f9219e..68602c51 100644
--- a/packages/plugins/vectorstore-meilisearch/src/index.ts
+++ b/packages/plugins/vectorstore-meilisearch/src/index.ts
@@ -87,6 +87,7 @@ class MeiliSearchVectorClient implements VectorStoreClient {
limit: options.limit ?? 10,
attributesToRetrieve: ["id"],
showRankingScore: true,
+ rankingScoreThreshold: options.rankingScoreThreshold,
});
return {
diff --git a/packages/shared-server/src/eventLogTypes.ts b/packages/shared-server/src/eventLogTypes.ts
index 40255ea3..1fd10a7a 100644
--- a/packages/shared-server/src/eventLogTypes.ts
+++ b/packages/shared-server/src/eventLogTypes.ts
@@ -81,6 +81,7 @@ type EventLogInternal =
| {
["event.name"]: "embeddingsWorker.run";
"bookmark.id"?: string;
+ "embedding.mode"?: "embed" | "index" | "delete";
"embedding.text_size"?: number;
"embedding.prompt_tokens"?: number;
"embedding.total_tokens"?: number;
diff --git a/packages/shared-server/src/queues.ts b/packages/shared-server/src/queues.ts
index c0456836..84882bb4 100644
--- a/packages/shared-server/src/queues.ts
+++ b/packages/shared-server/src/queues.ts
@@ -120,6 +120,10 @@ export function buildCrawlIdempotencyKey(payload: ZCrawlLinkRequest): string {
export const zOpenAIRequestSchema = z.object({
bookmarkId: z.string(),
type: z.enum(["summarize", "tag"]).default("tag"),
+ // Precomputed embedding so tagging can find similar bookmarks via
+ // search({vector}) without waiting for the vector to be indexed. Only set on
+ // the embed -> tag path.
+ embedding: z.array(z.number()).optional(),
});
export type ZOpenAIRequest = z.infer<typeof zOpenAIRequestSchema>;
@@ -131,12 +135,31 @@ export const OpenAIQueue = createDeferredQueue<ZOpenAIRequest>("openai_queue", {
});
// Embeddings Worker
-export const zEmbeddingsRequestSchema = z.object({
- bookmarkId: z.string(),
- type: z.enum(["index", "delete"]).default("index"),
- force: z.boolean().optional(),
- runTaggingOnComplete: z.boolean().optional(),
-});
+//
+// - "embed": entry point. Generates the bookmark embedding, then dispatches the
+// tagging job (carrying the vector) and an "index" job. Does NOT persist the
+// vector itself, so it never retries on indexing failures.
+// - "index": persists a precomputed vector in the vector store. Its retries (the
+// slow Meilisearch index build) are isolated and never re-trigger tagging.
+// - "delete": removes the vector from the store.
+export const zEmbeddingsRequestSchema = z.discriminatedUnion("type", [
+ z.object({
+ type: z.literal("embed"),
+ bookmarkId: z.string(),
+ force: z.boolean().optional(),
+ runTaggingOnComplete: z.boolean().optional().default(true),
+ }),
+ z.object({
+ type: z.literal("index"),
+ bookmarkId: z.string(),
+ userId: z.string(),
+ embedding: z.array(z.number()),
+ }),
+ z.object({
+ type: z.literal("delete"),
+ bookmarkId: z.string(),
+ }),
+]);
export type ZEmbeddingsRequest = z.infer<typeof zEmbeddingsRequestSchema>;
export const EmbeddingsQueue = createDeferredQueue<ZEmbeddingsRequest>(
diff --git a/packages/shared/vectorStore.ts b/packages/shared/vectorStore.ts
index 314af829..e7dbcee4 100644
--- a/packages/shared/vectorStore.ts
+++ b/packages/shared/vectorStore.ts
@@ -33,6 +33,8 @@ export interface VectorSearchOptions {
// Different filters are ANDed together
filter?: VectorFilterQuery[];
limit?: number;
+ // Drop hits whose ranking score is below this threshold (0..1)
+ rankingScoreThreshold?: number;
}
export interface VectorSimilarSearchOptions {
diff --git a/packages/trpc/routers/admin.ts b/packages/trpc/routers/admin.ts
index 8cf1cef5..07adc767 100644
--- a/packages/trpc/routers/admin.ts
+++ b/packages/trpc/routers/admin.ts
@@ -343,7 +343,7 @@ export const adminAppRouter = router({
EmbeddingsQueue.enqueue(
{
bookmarkId: id,
- type: "index",
+ type: "embed",
force: true,
runTaggingOnComplete: false,
},
@@ -856,7 +856,7 @@ export const adminAppRouter = router({
await EmbeddingsQueue.enqueue(
{
bookmarkId: input.bookmarkId,
- type: "index",
+ type: "embed",
force: true,
runTaggingOnComplete: false,
},
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts
index 24f78f85..ea628489 100644
--- a/packages/trpc/routers/bookmarks.ts
+++ b/packages/trpc/routers/bookmarks.ts
@@ -412,7 +412,8 @@ export const bookmarksAppRouter = router({
await EmbeddingsQueue.enqueue(
{
bookmarkId: bookmark.id,
- type: "index",
+ type: "embed",
+ runTaggingOnComplete: true,
},
enqueueOpts,
);