A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
feat: add extra instrumentation in the otel traces (#2453)
Mohamed Bassem538035c452bf
diff --git a/.gitignore b/.gitignoreindex e5aa93d8..88c50ccb 100644--- a/.gitignore+++ b/.gitignore@@ -66,3 +66,4 @@ auth_failures.log# Local directory for AI agent contexts.aicontext/+.codexdiff --git a/apps/workers/network.ts b/apps/workers/network.tsindex 4bc30562..2ef8483f 100644--- a/apps/workers/network.ts+++ b/apps/workers/network.ts@@ -86,6 +86,15 @@ function isAddressForbidden(address: string): boolean {return DISALLOWED_IP_RANGES.has(parsed.range());}+export function getBookmarkDomain(url?: string | null): string | undefined {+ if (!url) return undefined;+ try {+ return new URL(url).hostname;+ } catch {+ return undefined;+ }+}+export type UrlValidationResult =| { ok: true; url: URL }| { ok: false; reason: string };diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.tsindex 597d45d3..d3c20b7c 100644--- a/apps/workers/workers/crawlerWorker.ts+++ b/apps/workers/workers/crawlerWorker.ts@@ -30,6 +30,7 @@ import metascraperYoutube from "metascraper-youtube";import { crawlerStatusCodeCounter, workerStatsCounter } from "metrics";import {fetchWithProxy,+ getBookmarkDomain,getRandomProxy,matchesNoProxy,validateUrl,@@ -57,6 +58,7 @@ import {LinkCrawlerQueue,OpenAIQueue,QuotaService,+ setSpanAttributes,triggerSearchReindex,triggerWebhook,VideoWorkerQueue,@@ -435,7 +437,13 @@ async function browserlessCrawlPage(return await withSpan(tracer,"crawlerWorker.browserlessCrawlPage",- { attributes: { url, jobId } },+ {+ attributes: {+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ "job.id": jobId,+ },+ },async () => {logger.info(`[Crawler][${jobId}] Running in browserless mode. Will do a plain http request to "${url}". Screenshots will be disabled.`,@@ -473,7 +481,15 @@ async function crawlPage(return await withSpan(tracer,"crawlerWorker.crawlPage",- { attributes: { url, jobId, userId, forceStorePdf } },+ {+ attributes: {+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ "job.id": jobId,+ "user.id": userId,+ "crawler.forceStorePdf": forceStorePdf,+ },+ },async () => {// Check user's browser crawling settingconst userData = await db.query.users.findFirst({@@ -717,7 +733,13 @@ async function extractMetadata(return await withSpan(tracer,"crawlerWorker.extractMetadata",- { attributes: { url, jobId } },+ {+ attributes: {+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ "job.id": jobId,+ },+ },async () => {logger.info(`[Crawler][${jobId}] Will attempt to extract metadata from page ...`,@@ -745,7 +767,13 @@ async function extractReadableContent(return await withSpan(tracer,"crawlerWorker.extractReadableContent",- { attributes: { url, jobId } },+ {+ attributes: {+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ "job.id": jobId,+ },+ },async () => {logger.info(`[Crawler][${jobId}] Will attempt to extract readable content ...`,@@ -790,9 +818,9 @@ async function storeScreenshot("crawlerWorker.storeScreenshot",{attributes: {- jobId,- userId,- size: screenshot?.byteLength ?? 0,+ "job.id": jobId,+ "user.id": userId,+ "asset.size": screenshot?.byteLength ?? 0,},},async () => {@@ -849,9 +877,9 @@ async function storePdf("crawlerWorker.storePdf",{attributes: {- jobId,- userId,- size: pdf?.byteLength ?? 0,+ "job.id": jobId,+ "user.id": userId,+ "asset.size": pdf?.byteLength ?? 0,},},async () => {@@ -902,7 +930,15 @@ async function downloadAndStoreFile(return await withSpan(tracer,"crawlerWorker.downloadAndStoreFile",- { attributes: { url, jobId, userId, fileType } },+ {+ attributes: {+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ "job.id": jobId,+ "user.id": userId,+ "asset.type": fileType,+ },+ },async () => {let assetPath: string | undefined;try {@@ -1018,7 +1054,14 @@ async function archiveWebpage(return await withSpan(tracer,"crawlerWorker.archiveWebpage",- { attributes: { url, jobId, userId } },+ {+ attributes: {+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ "job.id": jobId,+ "user.id": userId,+ },+ },async () => {logger.info(`[Crawler][${jobId}] Will attempt to archive page ...`);const assetId = newAssetId();@@ -1103,7 +1146,13 @@ async function getContentType(return await withSpan(tracer,"crawlerWorker.getContentType",- { attributes: { url, jobId } },+ {+ attributes: {+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ "job.id": jobId,+ },+ },async () => {try {logger.info(@@ -1113,8 +1162,14 @@ async function getContentType(method: "GET",signal: AbortSignal.any([AbortSignal.timeout(5000), abortSignal]),});+ setSpanAttributes({+ "crawler.getContentType.statusCode": response.status,+ });const rawContentType = response.headers.get("content-type");const contentType = normalizeContentType(rawContentType);+ setSpanAttributes({+ "crawler.contentType": contentType ?? undefined,+ });logger.info(`[Crawler][${jobId}] Content-type for the url ${url} is "${contentType}"`,);@@ -1148,7 +1203,16 @@ async function handleAsAssetBookmark(return await withSpan(tracer,"crawlerWorker.handleAsAssetBookmark",- { attributes: { url, jobId, userId, bookmarkId, assetType } },+ {+ attributes: {+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ "job.id": jobId,+ "user.id": userId,+ "bookmark.id": bookmarkId,+ "asset.type": assetType,+ },+ },async () => {const downloaded = await downloadAndStoreFile(url,@@ -1218,9 +1282,11 @@ async function storeHtmlContent("crawlerWorker.storeHtmlContent",{attributes: {- jobId,- userId,- contentSize: htmlContent ? Buffer.byteLength(htmlContent, "utf8") : 0,+ "job.id": jobId,+ "user.id": userId,+ "bookmark.content.size": htmlContent+ ? Buffer.byteLength(htmlContent, "utf8")+ : 0,},},async () => {@@ -1302,13 +1368,14 @@ async function crawlAndParseUrl("crawlerWorker.crawlAndParseUrl",{attributes: {- url,- jobId,- userId,- bookmarkId,- archiveFullPage,- forceStorePdf,- hasPrecrawledArchive: !!precrawledArchiveAssetId,+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ "job.id": jobId,+ "user.id": userId,+ "bookmark.id": bookmarkId,+ "crawler.archiveFullPage": archiveFullPage,+ "crawler.forceStorePdf": forceStorePdf,+ "crawler.hasPrecrawledArchive": !!precrawledArchiveAssetId,},},async () => {@@ -1357,6 +1424,9 @@ async function crawlAndParseUrl(// Track status code in Prometheusif (statusCode !== null) {crawlerStatusCodeCounter.labels(statusCode.toString()).inc();+ setSpanAttributes({+ "crawler.statusCode": statusCode,+ });}const meta = await Promise.race([@@ -1576,7 +1646,13 @@ async function checkDomainRateLimit(url: string, jobId: string): Promise<void> {return await withSpan(tracer,"crawlerWorker.checkDomainRateLimit",- { attributes: { url, jobId } },+ {+ attributes: {+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ "job.id": jobId,+ },+ },async () => {const crawlerDomainRateLimitConfig =serverConfig.crawler.domainRatelimiting;diff --git a/apps/workers/workers/inference/summarize.ts b/apps/workers/workers/inference/summarize.tsindex 094c46ca..922eb5b7 100644--- a/apps/workers/workers/inference/summarize.ts+++ b/apps/workers/workers/inference/summarize.ts@@ -1,8 +1,13 @@import { and, eq } from "drizzle-orm";+import { getBookmarkDomain } from "network";import { db } from "@karakeep/db";import { bookmarks, customPrompts, users } from "@karakeep/db/schema";-import { triggerSearchReindex, ZOpenAIRequest } from "@karakeep/shared-server";+import {+ setSpanAttributes,+ triggerSearchReindex,+ ZOpenAIRequest,+} from "@karakeep/shared-server";import serverConfig from "@karakeep/shared/config";import { InferenceClient } from "@karakeep/shared/inference";import logger from "@karakeep/shared/logger";@@ -22,6 +27,7 @@ async function fetchBookmarkDetailsForSummary(bookmarkId: string) {description: true,htmlContent: true,contentAssetId: true,+ crawlStatusCode: true,publisher: true,author: true,url: true,@@ -65,6 +71,17 @@ export async function runSummarization(},});+ setSpanAttributes({+ "user.id": bookmarkData.userId,+ "bookmark.id": bookmarkData.id,+ "bookmark.url": bookmarkData.link?.url,+ "bookmark.domain": getBookmarkDomain(bookmarkData.link?.url),+ "bookmark.content.type": bookmarkData.type,+ "crawler.statusCode": bookmarkData.link?.crawlStatusCode ?? undefined,+ "inference.type": "summarization",+ "inference.model": serverConfig.inference.textModel,+ });+if (userSettings?.autoSummarizationEnabled === false) {logger.debug(`[inference][${jobId}] Skipping summarization job for bookmark with id "${bookmarkId}" because user has disabled auto-summarization.`,@@ -121,6 +138,10 @@ URL: ${link.url ?? ""}},});+ setSpanAttributes({+ "inference.prompt.customCount": prompts.length,+ });+const summaryPrompt = await buildSummaryPrompt(userSettings?.inferredTagLang ?? serverConfig.inference.inferredTagLang,prompts.map((p) => p.text),@@ -128,6 +149,10 @@ URL: ${link.url ?? ""}serverConfig.inference.contextLength,);+ setSpanAttributes({+ "inference.prompt.size": Buffer.byteLength(summaryPrompt, "utf8"),+ });+const summaryResult = await inferenceClient.inferFromText(summaryPrompt, {schema: null, // Summaries are typically free-form textabortSignal: job.abortSignal,@@ -139,6 +164,11 @@ URL: ${link.url ?? ""});}+ setSpanAttributes({+ "inference.summary.size": Buffer.byteLength(summaryResult.response, "utf8"),+ "inference.totalTokens": summaryResult.totalTokens,+ });+logger.info(`[inference][${jobId}] Generated summary for bookmark "${bookmarkId}" using ${summaryResult.totalTokens} tokens.`,);diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.tsindex 376eab14..b3006193 100644--- a/apps/workers/workers/inference/tagging.ts+++ b/apps/workers/workers/inference/tagging.ts@@ -1,4 +1,5 @@import { and, eq, inArray } from "drizzle-orm";+import { getBookmarkDomain } from "network";import { buildImpersonatingTRPCClient } from "trpc";import { z } from "zod";@@ -17,6 +18,7 @@ import {users,} from "@karakeep/db/schema";import {+ setSpanAttributes,triggerRuleEngineOnEvent,triggerSearchReindex,triggerWebhook,@@ -150,6 +152,9 @@ async function inferTagsFromImage(}const base64 = asset.toString("base64");+ setSpanAttributes({+ "inference.model": serverConfig.inference.imageModel,+ });return inferenceClient.inferFromImage(buildImagePrompt(inferredTagLang,@@ -176,6 +181,10 @@ async function fetchCustomPrompts(},});+ setSpanAttributes({+ "inference.prompt.customCount": prompts.length,+ });+let promptTexts = prompts.map((p) => p.text);if (containsTagsPlaceholder(prompts)) {promptTexts = await replaceTagsPlaceholders(promptTexts, userId);@@ -234,6 +243,12 @@ async function inferTagsFromPDF(serverConfig.inference.contextLength,tagStyle,);+ setSpanAttributes({+ "inference.model": serverConfig.inference.textModel,+ });+ setSpanAttributes({+ "inference.prompt.size": Buffer.byteLength(prompt, "utf8"),+ });return inferenceClient.inferFromText(prompt, {schema: openAIResponseSchema,abortSignal,@@ -251,6 +266,12 @@ async function inferTagsFromText(if (!prompt) {return null;}+ setSpanAttributes({+ "inference.model": serverConfig.inference.textModel,+ });+ setSpanAttributes({+ "inference.prompt.size": Buffer.byteLength(prompt, "utf8"),+ });return await inferenceClient.inferFromText(prompt, {schema: openAIResponseSchema,abortSignal,@@ -265,6 +286,18 @@ async function inferTags(tagStyle: ZTagStyle,inferredTagLang: string,) {+ setSpanAttributes({+ "user.id": bookmark.userId,+ "bookmark.id": bookmark.id,+ "bookmark.url": bookmark.link?.url,+ "bookmark.domain": getBookmarkDomain(bookmark.link?.url),+ "bookmark.content.type": bookmark.type,+ "crawler.statusCode": bookmark.link?.crawlStatusCode ?? undefined,+ "inference.tagging.style": tagStyle,+ "inference.lang": inferredTagLang,+ "inference.type": "tagging",+ });+let response: InferenceResponse | null;if (bookmark.link || bookmark.text) {response = await inferTagsFromText(@@ -325,6 +358,10 @@ async function inferTags(}return tag.trim();});+ setSpanAttributes({+ "inference.tagging.numGeneratedTags": tags.length,+ "inference.totalTokens": response.totalTokens,+ });return tags;} catch (e) {diff --git a/packages/shared-server/src/tracing.ts b/packages/shared-server/src/tracing.tsindex e831e019..10222f88 100644--- a/packages/shared-server/src/tracing.ts+++ b/packages/shared-server/src/tracing.ts@@ -24,6 +24,10 @@ import {import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";+import type { TracingAttributes } from "./tracingTypes";++export type { TracingAttributeKey, TracingAttributes } from "./tracingTypes";+let tracerProvider: NodeTracerProvider | null = null;let isInitialized = false;@@ -129,7 +133,7 @@ export async function withSpan<T>(spanName: string,options: {kind?: SpanKind;- attributes?: Record<string, string | number | boolean>;+ attributes?: TracingAttributes;},fn: (span: Span) => Promise<T>,): Promise<T> {@@ -168,7 +172,7 @@ export function withSpanSync<T>(spanName: string,options: {kind?: SpanKind;- attributes?: Record<string, string | number | boolean>;+ attributes?: TracingAttributes;},fn: (span: Span) => T,): T {@@ -213,9 +217,7 @@ export function addSpanEvent(/*** Set attributes on the current active span.*/-export function setSpanAttributes(- attributes: Record<string, string | number | boolean>,-): void {+export function setSpanAttributes(attributes: TracingAttributes): void {const span = getActiveSpan();if (span) {span.setAttributes(attributes);diff --git a/packages/shared-server/src/tracingTypes.ts b/packages/shared-server/src/tracingTypes.tsnew file mode 100644index 00000000..f397fa6f--- /dev/null+++ b/packages/shared-server/src/tracingTypes.ts@@ -0,0 +1,45 @@+export type TracingAttributeKey =+ // User attributes+ | "user.id"+ | "user.role"+ | "user.tier"+ // RPC attributes+ | "rpc.system"+ | "rpc.method"+ | "rpc.type"+ // Job attributes+ | "job.id"+ | "job.priority"+ | "job.runNumber"+ | "job.groupId"+ // Bookmark attributes+ | "bookmark.id"+ | "bookmark.url"+ | "bookmark.domain"+ | "bookmark.content.size"+ | "bookmark.content.type"+ // Asset attributes+ | "asset.id"+ | "asset.type"+ | "asset.size"+ // Crawler-specific attributes+ | "crawler.forceStorePdf"+ | "crawler.archiveFullPage"+ | "crawler.hasPrecrawledArchive"+ | "crawler.getContentType.statusCode"+ | "crawler.contentType"+ | "crawler.statusCode"+ // Inference-specific attributes+ | "inference.tagging.numGeneratedTags"+ | "inference.tagging.style"+ | "inference.summary.size"+ | "inference.lang"+ | "inference.prompt.size"+ | "inference.prompt.customCount"+ | "inference.totalTokens"+ | "inference.model"+ | "inference.type";++export type TracingAttributes = Partial<+ Record<TracingAttributeKey, string | number | boolean>+>;