A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix: fix plugin and stats registration in nextjs across reloads
Mohamed Bassema043c889d18c
diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsxindex 611ec93c..bcd4e7a9 100644--- a/apps/web/app/layout.tsx+++ b/apps/web/app/layout.tsx@@ -2,8 +2,6 @@ import type { Metadata } from "next";import { Inter } from "next/font/google";import { NuqsAdapter } from "nuqs/adapters/next/app";-import { loadAllPlugins } from "@karakeep/shared-server";-import "@karakeep/tailwind-config/globals.css";import type { Viewport } from "next";@@ -16,8 +14,6 @@ import { Toaster } from "sonner";import { clientConfig } from "@karakeep/shared/config";-await loadAllPlugins();-const inter = Inter({subsets: ["latin"],fallback: ["sans-serif"],diff --git a/apps/web/instrumentation.node.ts b/apps/web/instrumentation.node.tsindex 34b39e2e..72b88d2c 100644--- a/apps/web/instrumentation.node.ts+++ b/apps/web/instrumentation.node.ts@@ -1,4 +1,9 @@-import { initEventLogger, initTracing } from "@karakeep/shared-server";+import {+ initEventLogger,+ initTracing,+ loadAllPlugins,+} from "@karakeep/shared-server";+await loadAllPlugins();initTracing("web");initEventLogger("web");diff --git a/packages/api/index.ts b/packages/api/index.tsindex 1fa97b07..06b44888 100644--- a/packages/api/index.ts+++ b/packages/api/index.ts@@ -4,7 +4,6 @@ import { cors } from "hono/cors";import { logger as loggerMiddleware } from "hono/logger";import { poweredBy } from "hono/powered-by";-import { loadAllPlugins } from "@karakeep/shared-server";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";import { Context } from "@karakeep/trpc";@@ -27,8 +26,6 @@ import users from "./routes/users";import version from "./routes/version";import webhooks from "./routes/webhooks";-await loadAllPlugins();-const v1 = new Hono<{Variables: {ctx: Context;diff --git a/packages/shared-server/src/plugins.ts b/packages/shared-server/src/plugins.tsindex e5231a7b..0975323b 100644--- a/packages/shared-server/src/plugins.ts+++ b/packages/shared-server/src/plugins.ts@@ -1,18 +1,42 @@import { PluginManager } from "@karakeep/shared/plugins";-let pluginsLoaded = false;+const pluginLoaderStateKey = "__karakeep_plugins_loader_state__";++const globalPluginLoaderState = globalThis as typeof globalThis & {+ [pluginLoaderStateKey]?: {+ loaded: boolean;+ loading?: Promise<void>;+ };+};++const pluginLoaderState = (globalPluginLoaderState[pluginLoaderStateKey] ??= {+ loaded: false,+});+export async function loadAllPlugins() {- if (pluginsLoaded) {+ if (pluginLoaderState.loaded) {return;}- // Load plugins here. Order of plugin loading matter.- // Queue provider(s)- await import("@karakeep/plugins/queue-liteque");- await import("@karakeep/plugins/queue-restate");- await import("@karakeep/plugins/search-meilisearch");- // Rate limiters (order matters - last one wins)- await import("@karakeep/plugins/ratelimit-memory");- await import("@karakeep/plugins/ratelimit-redis");- PluginManager.logAllPlugins();- pluginsLoaded = true;+ if (pluginLoaderState.loading) {+ await pluginLoaderState.loading;+ return;+ }+ pluginLoaderState.loading = (async () => {+ // Load plugins here. Order of plugin loading matter.+ // Queue provider(s)+ await import("@karakeep/plugins/queue-liteque");+ await import("@karakeep/plugins/queue-restate");+ await import("@karakeep/plugins/search-meilisearch");+ // Rate limiters (order matters - last one wins)+ await import("@karakeep/plugins/ratelimit-memory");+ await import("@karakeep/plugins/ratelimit-redis");+ PluginManager.logAllPlugins();+ pluginLoaderState.loaded = true;+ })();++ try {+ await pluginLoaderState.loading;+ } finally {+ pluginLoaderState.loading = undefined;+ }}diff --git a/packages/shared-server/src/queues.ts b/packages/shared-server/src/queues.tsindex 51fd6d47..30bbff36 100644--- a/packages/shared-server/src/queues.ts+++ b/packages/shared-server/src/queues.ts@@ -9,7 +9,7 @@ import {} from "@karakeep/shared/queueing";import { zRuleEngineEventSchema } from "@karakeep/shared/types/rules";-import { loadAllPlugins } from ".";+import { loadAllPlugins } from "./plugins";export enum QueuePriority {Low = 50,diff --git a/packages/shared/plugins.ts b/packages/shared/plugins.tsindex 2d03ee39..5ceee91c 100644--- a/packages/shared/plugins.ts+++ b/packages/shared/plugins.ts@@ -30,15 +30,32 @@ export interface PluginProvider<T> {// Preserve the key-dependent value type: for K, store TPlugin<K>[]type ProviderMap = { [K in PluginType]: TPlugin<K>[] };-export class PluginManager {- private static providers: ProviderMap = {+const pluginProvidersKey = "__karakeep_plugins_providers__";++function createProviderMap(): ProviderMap {+ return {[PluginType.Search]: [],[PluginType.Queue]: [],[PluginType.RateLimit]: [],};+}++const globalPluginState = globalThis as typeof globalThis & {+ [pluginProvidersKey]?: ProviderMap;+};++export class PluginManager {+ private static providers: ProviderMap = (globalPluginState[+ pluginProvidersKey+ ] ??= createProviderMap());static register<T extends PluginType>(plugin: TPlugin<T>): void {- PluginManager.providers[plugin.type].push(plugin);+ const providers = PluginManager.providers[plugin.type];+ const existingProvider = providers.findIndex((p) => p.name === plugin.name);+ if (existingProvider >= 0) {+ return;+ }+ providers.push(plugin);}static async getClient<T extends PluginType>(diff --git a/packages/trpc/stats.ts b/packages/trpc/stats.tsindex b252af46..f4d4b19a 100644--- a/packages/trpc/stats.ts+++ b/packages/trpc/stats.ts@@ -1,5 +1,6 @@import { count, sum } from "drizzle-orm";import { Counter, Gauge, Histogram, register } from "prom-client";+import type { Metric } from "prom-client";import { db } from "@karakeep/db";import { assets, bookmarks, subscriptions, users } from "@karakeep/db/schema";@@ -18,166 +19,206 @@ import {} from "@karakeep/shared-server";import serverConfig from "@karakeep/shared/config";+function getOrCreateMetric<T extends Metric>(+ name: string,+ createMetric: () => T,+): T {+ return (register.getSingleMetric(name) as T | undefined) ?? createMetric();+}+// Queue metrics-const queuePendingJobsGauge = new Gauge({- name: "karakeep_queue_jobs",- help: "Number of jobs in each background queue",- labelNames: ["queue_name", "status"],- async collect() {- const queues = [- { name: "link_crawler", queue: LinkCrawlerQueue },- { name: "low_priority_crawler", queue: LowPriorityCrawlerQueue },- { name: "backup", queue: BackupQueue },- { name: "openai", queue: OpenAIQueue },- { name: "search_indexing", queue: SearchIndexingQueue },- { name: "admin_maintenance", queue: AdminMaintenanceQueue },- { name: "video_worker", queue: VideoWorkerQueue },- { name: "feed", queue: FeedQueue },- { name: "asset_preprocessing", queue: AssetPreprocessingQueue },- { name: "webhook", queue: WebhookQueue },- { name: "rule_engine", queue: RuleEngineQueue },- ];+const queuePendingJobsGauge = getOrCreateMetric(+ "karakeep_queue_jobs",+ () =>+ new Gauge({+ name: "karakeep_queue_jobs",+ help: "Number of jobs in each background queue",+ labelNames: ["queue_name", "status"],+ async collect() {+ const queues = [+ { name: "link_crawler", queue: LinkCrawlerQueue },+ { name: "low_priority_crawler", queue: LowPriorityCrawlerQueue },+ { name: "backup", queue: BackupQueue },+ { name: "openai", queue: OpenAIQueue },+ { name: "search_indexing", queue: SearchIndexingQueue },+ { name: "admin_maintenance", queue: AdminMaintenanceQueue },+ { name: "video_worker", queue: VideoWorkerQueue },+ { name: "feed", queue: FeedQueue },+ { name: "asset_preprocessing", queue: AssetPreprocessingQueue },+ { name: "webhook", queue: WebhookQueue },+ { name: "rule_engine", queue: RuleEngineQueue },+ ];- const stats = await Promise.all(- queues.map(async ({ name, queue }) => {- try {- return {- ...(await queue.stats()),- name,- };- } catch (error) {- console.error(`Failed to get stats for queue ${name}:`, error);- return { name, pending: 0, pending_retry: 0, failed: 0, running: 0 };- }- }),- );+ const stats = await Promise.all(+ queues.map(async ({ name, queue }) => {+ try {+ return {+ ...(await queue.stats()),+ name,+ };+ } catch (error) {+ console.error(`Failed to get stats for queue ${name}:`, error);+ return {+ name,+ pending: 0,+ pending_retry: 0,+ failed: 0,+ running: 0,+ };+ }+ }),+ );- stats.forEach(({ name, pending, pending_retry, failed, running }) => {- this.set({ queue_name: name, status: "pending" }, pending);- this.set({ queue_name: name, status: "pending_retry" }, pending_retry);- this.set({ queue_name: name, status: "failed" }, failed);- this.set({ queue_name: name, status: "running" }, running);- });- },-});+ stats.forEach(({ name, pending, pending_retry, failed, running }) => {+ this.set({ queue_name: name, status: "pending" }, pending);+ this.set(+ { queue_name: name, status: "pending_retry" },+ pending_retry,+ );+ this.set({ queue_name: name, status: "failed" }, failed);+ this.set({ queue_name: name, status: "running" }, running);+ });+ },+ }),+);// User metrics-const totalUsersGauge = new Gauge({- name: "karakeep_total_users",- help: "Total number of users in the system",- async collect() {- try {- const result = await db.select({ count: count() }).from(users);- this.set(result[0]?.count ?? 0);- } catch (error) {- console.error("Failed to get user count:", error);- this.set(0);- }- },-});--const subscriptionStatusGauge = serverConfig.stripe.isConfigured- ? new Gauge({- name: "karakeep_subscription_status",- help: "Total number of subscriptions per status",- labelNames: ["status", "tier"],+const totalUsersGauge = getOrCreateMetric(+ "karakeep_total_users",+ () =>+ new Gauge({+ name: "karakeep_total_users",+ help: "Total number of users in the system",async collect() {- this.reset();try {- const results = await db- .select({- status: subscriptions.status,- tier: subscriptions.tier,- count: count(),- })- .from(subscriptions)- .groupBy(subscriptions.status, subscriptions.tier);- for (const result of results) {- this.set(- { status: result.status, tier: result.tier },- result.count,- );- }+ const result = await db.select({ count: count() }).from(users);+ this.set(result[0]?.count ?? 0);} catch (error) {- console.error("Failed to get subscription status:", error);+ console.error("Failed to get user count:", error);+ this.set(0);}},- })- : null;+ }),+);++if (serverConfig.stripe.isConfigured) {+ getOrCreateMetric(+ "karakeep_subscription_status",+ () =>+ new Gauge({+ name: "karakeep_subscription_status",+ help: "Total number of subscriptions per status",+ labelNames: ["status", "tier"],+ async collect() {+ this.reset();+ try {+ const results = await db+ .select({+ status: subscriptions.status,+ tier: subscriptions.tier,+ count: count(),+ })+ .from(subscriptions)+ .groupBy(subscriptions.status, subscriptions.tier);+ for (const result of results) {+ this.set(+ { status: result.status, tier: result.tier },+ result.count,+ );+ }+ } catch (error) {+ console.error("Failed to get subscription status:", error);+ }+ },+ }),+ );+}// Asset metrics-const totalAssetSizeGauge = new Gauge({- name: "karakeep_total_asset_size_bytes",- help: "Total size of all assets in bytes",- async collect() {- try {- const result = await db- .select({ totalSize: sum(assets.size) })- .from(assets);- this.set(Number(result[0]?.totalSize ?? 0));- } catch (error) {- console.error("Failed to get total asset size:", error);- this.set(0);- }- },-});+const totalAssetSizeGauge = getOrCreateMetric(+ "karakeep_total_asset_size_bytes",+ () =>+ new Gauge({+ name: "karakeep_total_asset_size_bytes",+ help: "Total size of all assets in bytes",+ async collect() {+ try {+ const result = await db+ .select({ totalSize: sum(assets.size) })+ .from(assets);+ this.set(Number(result[0]?.totalSize ?? 0));+ } catch (error) {+ console.error("Failed to get total asset size:", error);+ this.set(0);+ }+ },+ }),+);// Bookmark metrics-const totalBookmarksGauge = new Gauge({- name: "karakeep_total_bookmarks",- help: "Total number of bookmarks in the system",- async collect() {- try {- const result = await db.select({ count: count() }).from(bookmarks);- this.set(result[0]?.count ?? 0);- } catch (error) {- console.error("Failed to get bookmark count:", error);- this.set(0);- }- },-});+const totalBookmarksGauge = getOrCreateMetric(+ "karakeep_total_bookmarks",+ () =>+ new Gauge({+ name: "karakeep_total_bookmarks",+ help: "Total number of bookmarks in the system",+ async collect() {+ try {+ const result = await db.select({ count: count() }).from(bookmarks);+ this.set(result[0]?.count ?? 0);+ } catch (error) {+ console.error("Failed to get bookmark count:", error);+ this.set(0);+ }+ },+ }),+);// Bookmark creation metrics-const bookmarkCreationCounter = new Counter({- name: "karakeep_bookmark_creations_total",- help: "Total number of bookmarks created",- labelNames: ["source"],-});+const bookmarkCreationCounter = getOrCreateMetric(+ "karakeep_bookmark_creations_total",+ () =>+ new Counter({+ name: "karakeep_bookmark_creations_total",+ help: "Total number of bookmarks created",+ labelNames: ["source"],+ }),+);// Api metrics-const apiRequestsTotalCounter = new Counter({- name: "karakeep_trpc_requests_total",- help: "Total number of API requests",- labelNames: ["type", "path", "is_error"],-});--const apiErrorsTotalCounter = new Counter({- name: "karakeep_trpc_errors_total",- help: "Total number of API requests",- labelNames: ["type", "path", "code"],-});+const apiRequestsTotalCounter = getOrCreateMetric(+ "karakeep_trpc_requests_total",+ () =>+ new Counter({+ name: "karakeep_trpc_requests_total",+ help: "Total number of API requests",+ labelNames: ["type", "path", "is_error"],+ }),+);-const apiRequestDurationSummary = new Histogram({- name: "karakeep_trpc_request_duration_seconds",- help: "Duration of tRPC requests in seconds",- labelNames: ["type", "path"],- buckets: [- 5e-3, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10,- ],-});+const apiErrorsTotalCounter = getOrCreateMetric(+ "karakeep_trpc_errors_total",+ () =>+ new Counter({+ name: "karakeep_trpc_errors_total",+ help: "Total number of API requests",+ labelNames: ["type", "path", "code"],+ }),+);-// Register all metrics-register.registerMetric(queuePendingJobsGauge);-register.registerMetric(totalUsersGauge);-if (subscriptionStatusGauge) {- register.registerMetric(subscriptionStatusGauge);-}-register.registerMetric(totalAssetSizeGauge);-register.registerMetric(totalBookmarksGauge);-register.registerMetric(bookmarkCreationCounter);-register.registerMetric(apiRequestsTotalCounter);-register.registerMetric(apiErrorsTotalCounter);-register.registerMetric(apiRequestDurationSummary);+const apiRequestDurationSummary = getOrCreateMetric(+ "karakeep_trpc_request_duration_seconds",+ () =>+ new Histogram({+ name: "karakeep_trpc_request_duration_seconds",+ help: "Duration of tRPC requests in seconds",+ labelNames: ["type", "path"],+ buckets: [+ 5e-3, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5,+ 10,+ ],+ }),+);export {queuePendingJobsGauge,