A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
feat: add embeddings-based potentially relevant tags to LLM context d… (#2858)
Mohamed Bassem2fbd77bda27f
diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.tsindex 1f303862..06ccbee7 100644--- a/apps/workers/workers/inference/tagging.ts+++ b/apps/workers/workers/inference/tagging.ts@@ -2,6 +2,7 @@ import { and, eq, inArray } from "drizzle-orm";import { getBookmarkDomain } from "network";import { buildImpersonatingTRPCClient } from "trpc";import { z } from "zod";+import { getVectorStoreClient } from "@karakeep/shared/vectorStore";import type { ZOpenAIRequest } from "@karakeep/shared-server";import type {@@ -32,6 +33,11 @@ import { RuleEngine } from "@karakeep/trpc/lib/ruleEngine";import { Bookmark } from "@karakeep/trpc/models/bookmarks";import { WebhooksService } from "@karakeep/trpc/models/webhooks.service";+/**+ * The maximum length of the relevant tag names to avoid bloating the inference context.+ */+const RELEVANT_TAG_TRUNCATE_LENGTH = 1000;+const openAIResponseSchema = z.object({tags: z.array(z.string()),});@@ -87,6 +93,7 @@ async function buildPrompt(tagStyle: ZTagStyle,inferredTagLang: string,curatedTags?: string[],+ potentialRelevantTags?: string[],): Promise<string | null> {const prompts = await fetchCustomPrompts(bookmark.userId, "text");if (bookmark.link) {@@ -113,6 +120,7 @@ Content: ${content ?? ""}`,serverConfig.inference.contextLength,tagStyle,curatedTags,+ potentialRelevantTags,);}@@ -124,6 +132,7 @@ Content: ${content ?? ""}`,serverConfig.inference.contextLength,tagStyle,curatedTags,+ potentialRelevantTags,);}@@ -138,6 +147,7 @@ async function inferTagsFromImage(tagStyle: ZTagStyle,inferredTagLang: string,curatedTags?: string[],+ potentialRelevantTags?: string[],): Promise<InferenceResponse | null> {const { asset, metadata } = await readAsset({userId: bookmark.userId,@@ -166,6 +176,7 @@ async function inferTagsFromImage(await fetchCustomPrompts(bookmark.userId, "images"),tagStyle,curatedTags,+ potentialRelevantTags,),metadata.contentType,base64,@@ -242,6 +253,7 @@ async function inferTagsFromPDF(tagStyle: ZTagStyle,inferredTagLang: string,curatedTags?: string[],+ potentialRelevantTags?: string[],) {const prompt = await buildTextPrompt(inferredTagLang,@@ -250,6 +262,7 @@ async function inferTagsFromPDF(serverConfig.inference.contextLength,tagStyle,curatedTags,+ potentialRelevantTags,);addLogFields<"inferenceWorker.run">({"inference.model": serverConfig.inference.textModel,@@ -268,12 +281,14 @@ async function inferTagsFromText(tagStyle: ZTagStyle,inferredTagLang: string,curatedTags?: string[],+ potentialRelevantTags?: string[],) {const prompt = await buildPrompt(bookmark,tagStyle,inferredTagLang,curatedTags,+ potentialRelevantTags,);if (!prompt) {return null;@@ -296,6 +311,7 @@ async function inferTags(tagStyle: ZTagStyle,inferredTagLang: string,curatedTags?: string[],+ potentialRelevantTags?: string[],) {setSpanAttributes({"user.id": bookmark.userId,@@ -310,6 +326,8 @@ async function inferTags("crawler.status_code": bookmark.link?.crawlStatusCode ?? undefined,"inference.tagging.style": tagStyle,"inference.tagging.lang": inferredTagLang,+ "inference.tagging.num_potential_relevant_tags":+ potentialRelevantTags?.length ?? 0,});let response: InferenceResponse | null;@@ -321,6 +339,7 @@ async function inferTags(tagStyle,inferredTagLang,curatedTags,+ potentialRelevantTags,);} else if (bookmark.asset) {switch (bookmark.asset.assetType) {@@ -333,6 +352,7 @@ async function inferTags(tagStyle,inferredTagLang,curatedTags,+ potentialRelevantTags,);break;case "pdf":@@ -344,6 +364,7 @@ async function inferTags(tagStyle,inferredTagLang,curatedTags,+ potentialRelevantTags,);break;default:@@ -502,6 +523,69 @@ async function fetchBookmark(linkId: string) {});}+/**+ * Finds potentially relevant tags for the passed bookmarkId by find similar bookmarks and fetching their tags.+ */+async function getPotentiallyRelevantTags(+ jobId: string,+ bookmarkId: string,+ userId: string,+): Promise<string[] | null> {+ const client = await getVectorStoreClient();+ if (!client) {+ return null;+ }+ const similarBookmarkIds = await client+ .findSimilar({+ id: bookmarkId,+ limit: 10,+ filter: [+ {+ type: "eq",+ field: "userId",+ value: userId,+ },+ ],+ })+ .then((r) => r.hits.map((r) => r.id));++ if (similarBookmarkIds.length === 0) {+ return null;+ }++ const tags = await db+ .selectDistinct({ name: bookmarkTags.name })+ .from(bookmarkTags)+ .leftJoin(tagsOnBookmarks, eq(bookmarkTags.id, tagsOnBookmarks.tagId))+ .where(inArray(tagsOnBookmarks.bookmarkId, similarBookmarkIds))+ .limit(100);++ // Let's try to use shorter tags first+ tags.sort((a, b) => a.name.length - b.name.length);++ const toKeep = [];+ let lengthSoFar = 0;+ for (const tag of tags) {+ // Account for the ", " separator between tags in the joined prompt output+ const separatorLen = toKeep.length > 0 ? 2 : 0;++ if (+ lengthSoFar + separatorLen + tag.name.length >+ RELEVANT_TAG_TRUNCATE_LENGTH+ ) {+ break;+ }+ toKeep.push(tag.name);+ lengthSoFar += tag.name.length;+ }++ logger.debug(+ `[inference][${jobId}] Will use ${toKeep.length} potential tags (out of ${tags.length}, across ${similarBookmarkIds.length} bookmarks) for the bookmark with id ${bookmarkId}: ${toKeep.join(", ")}`,+ );++ return [...toKeep];+}+export async function runTagging(bookmarkId: string,job: DequeuedJob<ZOpenAIRequest>,@@ -541,6 +625,7 @@ export async function runTagging(// Resolve curated tag names if configuredlet curatedTagNames: string[] | undefined;+ let potentialRelevantTags: string[] | undefined = undefined;if (userSettings?.curatedTagIds && userSettings.curatedTagIds.length > 0) {const tags = await db.query.bookmarkTags.findMany({where: and(@@ -550,6 +635,20 @@ export async function runTagging(columns: { name: true },});curatedTagNames = tags.map((t) => t.name);+ } else {+ // If no curated tags are configured, try to find some potentially relevant tags+ try {+ potentialRelevantTags =+ (await getPotentiallyRelevantTags(+ jobId,+ bookmarkId,+ bookmark.userId,+ )) ?? undefined;+ } catch (e) {+ logger.error(+ `[inference][${jobId}] Failed to find potentially relevant tags: ${e}`,+ );+ }}logger.info(@@ -564,6 +663,7 @@ export async function runTagging(userSettings?.tagStyle ?? "as-generated",userSettings?.inferredTagLang ?? serverConfig.inference.inferredTagLang,curatedTagNames,+ potentialRelevantTags,);if (tags === null) {diff --git a/packages/plugins/vectorstore-meilisearch/src/index.ts b/packages/plugins/vectorstore-meilisearch/src/index.tsindex d0587577..96f9219e 100644--- a/packages/plugins/vectorstore-meilisearch/src/index.ts+++ b/packages/plugins/vectorstore-meilisearch/src/index.ts@@ -7,6 +7,7 @@ import type {VectorSearchOptions,VectorSearchResponse,VectorStoreClient,+ VectorSimilarSearchOptions,} from "@karakeep/shared/vectorStore";import serverConfig from "@karakeep/shared/config";import { PluginProvider } from "@karakeep/shared/plugins";@@ -97,6 +98,28 @@ class MeiliSearchVectorClient implements VectorStoreClient {};}+ async findSimilar(+ options: VectorSimilarSearchOptions,+ ): Promise<VectorSearchResponse> {+ const results = await this.index.searchSimilarDocuments({+ id: options.id,+ filter: options.filter?.map((f) => filterToMeiliSearchFilter(f)),+ limit: options.limit ?? 10,+ attributesToRetrieve: ["id"],+ showRankingScore: true,+ embedder: "default",+ rankingScoreThreshold: 0.75,+ });++ return {+ hits: results.hits.map((hit) => ({+ id: hit.id,+ score: hit._rankingScore ?? 0,+ })),+ processingTimeMs: results.processingTimeMs,+ };+ }+async clearIndex(): Promise<void> {const task = await this.index.deleteAllDocuments();await this.ensureTaskSuccess(task.taskUid);diff --git a/packages/shared-server/src/eventLogTypes.ts b/packages/shared-server/src/eventLogTypes.tsindex 6ce13353..40255ea3 100644--- a/packages/shared-server/src/eventLogTypes.ts+++ b/packages/shared-server/src/eventLogTypes.ts@@ -23,6 +23,7 @@ type EventLogInternal ="inference.tagging.style"?: string;"inference.tagging.lang"?: string;"inference.tagging.num_generated_tags"?: number;+ "inference.tagging.num_potential_relevant_tags"?: number;}| {["event.name"]: "bookmark.summarize";diff --git a/packages/shared/prompts.server.ts b/packages/shared/prompts.server.tsindex c53f4190..86978fe7 100644--- a/packages/shared/prompts.server.ts+++ b/packages/shared/prompts.server.ts@@ -50,6 +50,7 @@ export async function buildTextPrompt(contextLength: number,tagStyle: ZTagStyle,curatedTags?: string[],+ potentialRelevantTags?: string[],): Promise<string> {content = preprocessContent(content);const promptTemplate = constructTextTaggingPrompt(@@ -58,6 +59,7 @@ export async function buildTextPrompt("",tagStyle,curatedTags,+ potentialRelevantTags,);const promptSize = await calculateNumTokens(promptTemplate);const available = Math.max(0, contextLength - promptSize);@@ -69,6 +71,7 @@ export async function buildTextPrompt(truncatedContent,tagStyle,curatedTags,+ potentialRelevantTags,);}diff --git a/packages/shared/prompts.ts b/packages/shared/prompts.tsindex 6c5c02c4..0c800210 100644--- a/packages/shared/prompts.ts+++ b/packages/shared/prompts.ts@@ -1,5 +1,9 @@import type { ZTagStyle } from "./types/users";-import { getCuratedTagsPrompt, getTagStylePrompt } from "./utils/tag";+import {+ getCuratedTagsPrompt,+ getTagStylePrompt,+ getPotentialRelevantTagsPrompt,+} from "./utils/tag";/*** Remove duplicate whitespaces to avoid tokenization issues@@ -13,9 +17,13 @@ export function buildImagePrompt(customPrompts: string[],tagStyle: ZTagStyle,curatedTags?: string[],+ potentialRelevantTags?: string[],) {const tagStyleInstruction = getTagStylePrompt(tagStyle);const curatedInstruction = getCuratedTagsPrompt(curatedTags);+ const potentialRelevantTagsInstruction = getPotentialRelevantTagsPrompt(+ potentialRelevantTags,+ );return `You are an expert whose responsibility is to help with automatic text tagging for a read-it-later/bookmarking app.@@ -26,6 +34,7 @@ Analyze the attached image and suggest relevant tags that describe its key theme- Aim for 10-15 tags.- If there are no good tags, don't emit any.${curatedInstruction}+${potentialRelevantTagsInstruction}${tagStyleInstruction}${customPrompts && customPrompts.map((p) => `- ${p}`).join("\n")}You must respond in valid JSON with the key "tags" and the value is list of tags. Don't wrap the response in a markdown code.`;@@ -40,9 +49,13 @@ export function constructTextTaggingPrompt(content: string,tagStyle: ZTagStyle,curatedTags?: string[],+ potentialRelevantTags?: string[],): string {const tagStyleInstruction = getTagStylePrompt(tagStyle);const curatedInstruction = getCuratedTagsPrompt(curatedTags);+ const potentialRelevantTagsInstruction = getPotentialRelevantTagsPrompt(+ potentialRelevantTags,+ );return `You are an expert whose responsibility is to help with automatic tagging for a read-it-later/bookmarking app.@@ -56,6 +69,7 @@ Analyze the TEXT_CONTENT below and suggest relevant tags that describe its key t- Aim for 3-5 tags.- If there are no good tags, leave the array empty.${curatedInstruction}+${potentialRelevantTagsInstruction}${tagStyleInstruction}${customPrompts && customPrompts.map((p) => `- ${p}`).join("\n")}diff --git a/packages/shared/utils/tag.ts b/packages/shared/utils/tag.tsindex b69b817e..a8aeae7f 100644--- a/packages/shared/utils/tag.ts+++ b/packages/shared/utils/tag.ts@@ -35,3 +35,12 @@ export function getCuratedTagsPrompt(curatedTags?: string[]): string {}return "";}++export function getPotentialRelevantTagsPrompt(+ potentialRelevantTags?: string[],+): string {+ if (potentialRelevantTags && potentialRelevantTags.length > 0) {+ return `- Similar bookmarks were tagged with the following tags (reuse if possible, ignore if irrelevant): ${potentialRelevantTags.join(", ")}`;+ }+ return "";+}diff --git a/packages/shared/vectorStore.ts b/packages/shared/vectorStore.tsindex ed17752d..314af829 100644--- a/packages/shared/vectorStore.ts+++ b/packages/shared/vectorStore.ts@@ -35,6 +35,13 @@ export interface VectorSearchOptions {limit?: number;}+export interface VectorSimilarSearchOptions {+ id: string;+ // Different filters are ANDed together+ filter?: VectorFilterQuery[];+ limit?: number;+}+export interface VectorSearchResponse {hits: VectorSearchResult[];processingTimeMs: number;@@ -56,6 +63,13 @@ export interface VectorStoreClient {*/search(options: VectorSearchOptions): Promise<VectorSearchResponse>;+ /**+ * Search for bookmarks similar to the passed bookmarkId+ */+ findSimilar(+ options: VectorSimilarSearchOptions,+ ): Promise<VectorSearchResponse>;+/*** Clear all vectors from the index*/