Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

feat: add embeddings-based potentially relevant tags to LLM context d… (#2858)

Mohamed Bassem2fbd77bda27f
diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.ts
index 1f303862..06ccbee7 100644
--- a/apps/workers/workers/inference/tagging.ts
+++ b/apps/workers/workers/inference/tagging.ts
@@ -2,6 +2,7 @@ import { and, eq, inArray } from "drizzle-orm";
import { getBookmarkDomain } from "network";
import { buildImpersonatingTRPCClient } from "trpc";
import { z } from "zod";
+import { getVectorStoreClient } from "@karakeep/shared/vectorStore";
import type { ZOpenAIRequest } from "@karakeep/shared-server";
import type {
@@ -32,6 +33,11 @@ import { RuleEngine } from "@karakeep/trpc/lib/ruleEngine";
import { Bookmark } from "@karakeep/trpc/models/bookmarks";
import { WebhooksService } from "@karakeep/trpc/models/webhooks.service";
+/**
+ * The maximum length of the relevant tag names to avoid bloating the inference context.
+ */
+const RELEVANT_TAG_TRUNCATE_LENGTH = 1000;
+
const openAIResponseSchema = z.object({
tags: z.array(z.string()),
});
@@ -87,6 +93,7 @@ async function buildPrompt(
tagStyle: ZTagStyle,
inferredTagLang: string,
curatedTags?: string[],
+ potentialRelevantTags?: string[],
): Promise<string | null> {
const prompts = await fetchCustomPrompts(bookmark.userId, "text");
if (bookmark.link) {
@@ -113,6 +120,7 @@ Content: ${content ?? ""}`,
serverConfig.inference.contextLength,
tagStyle,
curatedTags,
+ potentialRelevantTags,
);
}
@@ -124,6 +132,7 @@ Content: ${content ?? ""}`,
serverConfig.inference.contextLength,
tagStyle,
curatedTags,
+ potentialRelevantTags,
);
}
@@ -138,6 +147,7 @@ async function inferTagsFromImage(
tagStyle: ZTagStyle,
inferredTagLang: string,
curatedTags?: string[],
+ potentialRelevantTags?: string[],
): Promise<InferenceResponse | null> {
const { asset, metadata } = await readAsset({
userId: bookmark.userId,
@@ -166,6 +176,7 @@ async function inferTagsFromImage(
await fetchCustomPrompts(bookmark.userId, "images"),
tagStyle,
curatedTags,
+ potentialRelevantTags,
),
metadata.contentType,
base64,
@@ -242,6 +253,7 @@ async function inferTagsFromPDF(
tagStyle: ZTagStyle,
inferredTagLang: string,
curatedTags?: string[],
+ potentialRelevantTags?: string[],
) {
const prompt = await buildTextPrompt(
inferredTagLang,
@@ -250,6 +262,7 @@ async function inferTagsFromPDF(
serverConfig.inference.contextLength,
tagStyle,
curatedTags,
+ potentialRelevantTags,
);
addLogFields<"inferenceWorker.run">({
"inference.model": serverConfig.inference.textModel,
@@ -268,12 +281,14 @@ async function inferTagsFromText(
tagStyle: ZTagStyle,
inferredTagLang: string,
curatedTags?: string[],
+ potentialRelevantTags?: string[],
) {
const prompt = await buildPrompt(
bookmark,
tagStyle,
inferredTagLang,
curatedTags,
+ potentialRelevantTags,
);
if (!prompt) {
return null;
@@ -296,6 +311,7 @@ async function inferTags(
tagStyle: ZTagStyle,
inferredTagLang: string,
curatedTags?: string[],
+ potentialRelevantTags?: string[],
) {
setSpanAttributes({
"user.id": bookmark.userId,
@@ -310,6 +326,8 @@ async function inferTags(
"crawler.status_code": bookmark.link?.crawlStatusCode ?? undefined,
"inference.tagging.style": tagStyle,
"inference.tagging.lang": inferredTagLang,
+ "inference.tagging.num_potential_relevant_tags":
+ potentialRelevantTags?.length ?? 0,
});
let response: InferenceResponse | null;
@@ -321,6 +339,7 @@ async function inferTags(
tagStyle,
inferredTagLang,
curatedTags,
+ potentialRelevantTags,
);
} else if (bookmark.asset) {
switch (bookmark.asset.assetType) {
@@ -333,6 +352,7 @@ async function inferTags(
tagStyle,
inferredTagLang,
curatedTags,
+ potentialRelevantTags,
);
break;
case "pdf":
@@ -344,6 +364,7 @@ async function inferTags(
tagStyle,
inferredTagLang,
curatedTags,
+ potentialRelevantTags,
);
break;
default:
@@ -502,6 +523,69 @@ async function fetchBookmark(linkId: string) {
});
}
+/**
+ * Finds potentially relevant tags for the passed bookmarkId by find similar bookmarks and fetching their tags.
+ */
+async function getPotentiallyRelevantTags(
+ jobId: string,
+ bookmarkId: string,
+ userId: string,
+): Promise<string[] | null> {
+ const client = await getVectorStoreClient();
+ if (!client) {
+ return null;
+ }
+ const similarBookmarkIds = await client
+ .findSimilar({
+ id: bookmarkId,
+ limit: 10,
+ filter: [
+ {
+ type: "eq",
+ field: "userId",
+ value: userId,
+ },
+ ],
+ })
+ .then((r) => r.hits.map((r) => r.id));
+
+ if (similarBookmarkIds.length === 0) {
+ return null;
+ }
+
+ const tags = await db
+ .selectDistinct({ name: bookmarkTags.name })
+ .from(bookmarkTags)
+ .leftJoin(tagsOnBookmarks, eq(bookmarkTags.id, tagsOnBookmarks.tagId))
+ .where(inArray(tagsOnBookmarks.bookmarkId, similarBookmarkIds))
+ .limit(100);
+
+ // Let's try to use shorter tags first
+ tags.sort((a, b) => a.name.length - b.name.length);
+
+ const toKeep = [];
+ let lengthSoFar = 0;
+ for (const tag of tags) {
+ // Account for the ", " separator between tags in the joined prompt output
+ const separatorLen = toKeep.length > 0 ? 2 : 0;
+
+ if (
+ lengthSoFar + separatorLen + tag.name.length >
+ RELEVANT_TAG_TRUNCATE_LENGTH
+ ) {
+ break;
+ }
+ toKeep.push(tag.name);
+ lengthSoFar += tag.name.length;
+ }
+
+ logger.debug(
+ `[inference][${jobId}] Will use ${toKeep.length} potential tags (out of ${tags.length}, across ${similarBookmarkIds.length} bookmarks) for the bookmark with id ${bookmarkId}: ${toKeep.join(", ")}`,
+ );
+
+ return [...toKeep];
+}
+
export async function runTagging(
bookmarkId: string,
job: DequeuedJob<ZOpenAIRequest>,
@@ -541,6 +625,7 @@ export async function runTagging(
// Resolve curated tag names if configured
let curatedTagNames: string[] | undefined;
+ let potentialRelevantTags: string[] | undefined = undefined;
if (userSettings?.curatedTagIds && userSettings.curatedTagIds.length > 0) {
const tags = await db.query.bookmarkTags.findMany({
where: and(
@@ -550,6 +635,20 @@ export async function runTagging(
columns: { name: true },
});
curatedTagNames = tags.map((t) => t.name);
+ } else {
+ // If no curated tags are configured, try to find some potentially relevant tags
+ try {
+ potentialRelevantTags =
+ (await getPotentiallyRelevantTags(
+ jobId,
+ bookmarkId,
+ bookmark.userId,
+ )) ?? undefined;
+ } catch (e) {
+ logger.error(
+ `[inference][${jobId}] Failed to find potentially relevant tags: ${e}`,
+ );
+ }
}
logger.info(
@@ -564,6 +663,7 @@ export async function runTagging(
userSettings?.tagStyle ?? "as-generated",
userSettings?.inferredTagLang ?? serverConfig.inference.inferredTagLang,
curatedTagNames,
+ potentialRelevantTags,
);
if (tags === null) {
diff --git a/packages/plugins/vectorstore-meilisearch/src/index.ts b/packages/plugins/vectorstore-meilisearch/src/index.ts
index d0587577..96f9219e 100644
--- a/packages/plugins/vectorstore-meilisearch/src/index.ts
+++ b/packages/plugins/vectorstore-meilisearch/src/index.ts
@@ -7,6 +7,7 @@ import type {
VectorSearchOptions,
VectorSearchResponse,
VectorStoreClient,
+ VectorSimilarSearchOptions,
} from "@karakeep/shared/vectorStore";
import serverConfig from "@karakeep/shared/config";
import { PluginProvider } from "@karakeep/shared/plugins";
@@ -97,6 +98,28 @@ class MeiliSearchVectorClient implements VectorStoreClient {
};
}
+ async findSimilar(
+ options: VectorSimilarSearchOptions,
+ ): Promise<VectorSearchResponse> {
+ const results = await this.index.searchSimilarDocuments({
+ id: options.id,
+ filter: options.filter?.map((f) => filterToMeiliSearchFilter(f)),
+ limit: options.limit ?? 10,
+ attributesToRetrieve: ["id"],
+ showRankingScore: true,
+ embedder: "default",
+ rankingScoreThreshold: 0.75,
+ });
+
+ return {
+ hits: results.hits.map((hit) => ({
+ id: hit.id,
+ score: hit._rankingScore ?? 0,
+ })),
+ processingTimeMs: results.processingTimeMs,
+ };
+ }
+
async clearIndex(): Promise<void> {
const task = await this.index.deleteAllDocuments();
await this.ensureTaskSuccess(task.taskUid);
diff --git a/packages/shared-server/src/eventLogTypes.ts b/packages/shared-server/src/eventLogTypes.ts
index 6ce13353..40255ea3 100644
--- a/packages/shared-server/src/eventLogTypes.ts
+++ b/packages/shared-server/src/eventLogTypes.ts
@@ -23,6 +23,7 @@ type EventLogInternal =
"inference.tagging.style"?: string;
"inference.tagging.lang"?: string;
"inference.tagging.num_generated_tags"?: number;
+ "inference.tagging.num_potential_relevant_tags"?: number;
}
| {
["event.name"]: "bookmark.summarize";
diff --git a/packages/shared/prompts.server.ts b/packages/shared/prompts.server.ts
index c53f4190..86978fe7 100644
--- a/packages/shared/prompts.server.ts
+++ b/packages/shared/prompts.server.ts
@@ -50,6 +50,7 @@ export async function buildTextPrompt(
contextLength: number,
tagStyle: ZTagStyle,
curatedTags?: string[],
+ potentialRelevantTags?: string[],
): Promise<string> {
content = preprocessContent(content);
const promptTemplate = constructTextTaggingPrompt(
@@ -58,6 +59,7 @@ export async function buildTextPrompt(
"",
tagStyle,
curatedTags,
+ potentialRelevantTags,
);
const promptSize = await calculateNumTokens(promptTemplate);
const available = Math.max(0, contextLength - promptSize);
@@ -69,6 +71,7 @@ export async function buildTextPrompt(
truncatedContent,
tagStyle,
curatedTags,
+ potentialRelevantTags,
);
}
diff --git a/packages/shared/prompts.ts b/packages/shared/prompts.ts
index 6c5c02c4..0c800210 100644
--- a/packages/shared/prompts.ts
+++ b/packages/shared/prompts.ts
@@ -1,5 +1,9 @@
import type { ZTagStyle } from "./types/users";
-import { getCuratedTagsPrompt, getTagStylePrompt } from "./utils/tag";
+import {
+ getCuratedTagsPrompt,
+ getTagStylePrompt,
+ getPotentialRelevantTagsPrompt,
+} from "./utils/tag";
/**
* Remove duplicate whitespaces to avoid tokenization issues
@@ -13,9 +17,13 @@ export function buildImagePrompt(
customPrompts: string[],
tagStyle: ZTagStyle,
curatedTags?: string[],
+ potentialRelevantTags?: string[],
) {
const tagStyleInstruction = getTagStylePrompt(tagStyle);
const curatedInstruction = getCuratedTagsPrompt(curatedTags);
+ const potentialRelevantTagsInstruction = getPotentialRelevantTagsPrompt(
+ potentialRelevantTags,
+ );
return `
You are an expert whose responsibility is to help with automatic text tagging for a read-it-later/bookmarking app.
@@ -26,6 +34,7 @@ Analyze the attached image and suggest relevant tags that describe its key theme
- Aim for 10-15 tags.
- If there are no good tags, don't emit any.
${curatedInstruction}
+${potentialRelevantTagsInstruction}
${tagStyleInstruction}
${customPrompts && customPrompts.map((p) => `- ${p}`).join("\n")}
You must respond in valid JSON with the key "tags" and the value is list of tags. Don't wrap the response in a markdown code.`;
@@ -40,9 +49,13 @@ export function constructTextTaggingPrompt(
content: string,
tagStyle: ZTagStyle,
curatedTags?: string[],
+ potentialRelevantTags?: string[],
): string {
const tagStyleInstruction = getTagStylePrompt(tagStyle);
const curatedInstruction = getCuratedTagsPrompt(curatedTags);
+ const potentialRelevantTagsInstruction = getPotentialRelevantTagsPrompt(
+ potentialRelevantTags,
+ );
return `
You are an expert whose responsibility is to help with automatic tagging for a read-it-later/bookmarking app.
@@ -56,6 +69,7 @@ Analyze the TEXT_CONTENT below and suggest relevant tags that describe its key t
- Aim for 3-5 tags.
- If there are no good tags, leave the array empty.
${curatedInstruction}
+${potentialRelevantTagsInstruction}
${tagStyleInstruction}
${customPrompts && customPrompts.map((p) => `- ${p}`).join("\n")}
diff --git a/packages/shared/utils/tag.ts b/packages/shared/utils/tag.ts
index b69b817e..a8aeae7f 100644
--- a/packages/shared/utils/tag.ts
+++ b/packages/shared/utils/tag.ts
@@ -35,3 +35,12 @@ export function getCuratedTagsPrompt(curatedTags?: string[]): string {
}
return "";
}
+
+export function getPotentialRelevantTagsPrompt(
+ potentialRelevantTags?: string[],
+): string {
+ if (potentialRelevantTags && potentialRelevantTags.length > 0) {
+ return `- Similar bookmarks were tagged with the following tags (reuse if possible, ignore if irrelevant): ${potentialRelevantTags.join(", ")}`;
+ }
+ return "";
+}
diff --git a/packages/shared/vectorStore.ts b/packages/shared/vectorStore.ts
index ed17752d..314af829 100644
--- a/packages/shared/vectorStore.ts
+++ b/packages/shared/vectorStore.ts
@@ -35,6 +35,13 @@ export interface VectorSearchOptions {
limit?: number;
}
+export interface VectorSimilarSearchOptions {
+ id: string;
+ // Different filters are ANDed together
+ filter?: VectorFilterQuery[];
+ limit?: number;
+}
+
export interface VectorSearchResponse {
hits: VectorSearchResult[];
processingTimeMs: number;
@@ -56,6 +63,13 @@ export interface VectorStoreClient {
*/
search(options: VectorSearchOptions): Promise<VectorSearchResponse>;
+ /**
+ * Search for bookmarks similar to the passed bookmarkId
+ */
+ findSimilar(
+ options: VectorSimilarSearchOptions,
+ ): Promise<VectorSearchResponse>;
+
/**
* Clear all vectors from the index
*/