Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

fix: fix plugin and stats registration in nextjs across reloads

Mohamed Bassema043c889d18c
diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx
index 611ec93c..bcd4e7a9 100644
--- a/apps/web/app/layout.tsx
+++ b/apps/web/app/layout.tsx
@@ -2,8 +2,6 @@ import type { Metadata } from "next";
import { Inter } from "next/font/google";
import { NuqsAdapter } from "nuqs/adapters/next/app";
-import { loadAllPlugins } from "@karakeep/shared-server";
-
import "@karakeep/tailwind-config/globals.css";
import type { Viewport } from "next";
@@ -16,8 +14,6 @@ import { Toaster } from "sonner";
import { clientConfig } from "@karakeep/shared/config";
-await loadAllPlugins();
-
const inter = Inter({
subsets: ["latin"],
fallback: ["sans-serif"],
diff --git a/apps/web/instrumentation.node.ts b/apps/web/instrumentation.node.ts
index 34b39e2e..72b88d2c 100644
--- a/apps/web/instrumentation.node.ts
+++ b/apps/web/instrumentation.node.ts
@@ -1,4 +1,9 @@
-import { initEventLogger, initTracing } from "@karakeep/shared-server";
+import {
+ initEventLogger,
+ initTracing,
+ loadAllPlugins,
+} from "@karakeep/shared-server";
+await loadAllPlugins();
initTracing("web");
initEventLogger("web");
diff --git a/packages/api/index.ts b/packages/api/index.ts
index 1fa97b07..06b44888 100644
--- a/packages/api/index.ts
+++ b/packages/api/index.ts
@@ -4,7 +4,6 @@ import { cors } from "hono/cors";
import { logger as loggerMiddleware } from "hono/logger";
import { poweredBy } from "hono/powered-by";
-import { loadAllPlugins } from "@karakeep/shared-server";
import serverConfig from "@karakeep/shared/config";
import logger from "@karakeep/shared/logger";
import { Context } from "@karakeep/trpc";
@@ -27,8 +26,6 @@ import users from "./routes/users";
import version from "./routes/version";
import webhooks from "./routes/webhooks";
-await loadAllPlugins();
-
const v1 = new Hono<{
Variables: {
ctx: Context;
diff --git a/packages/shared-server/src/plugins.ts b/packages/shared-server/src/plugins.ts
index e5231a7b..0975323b 100644
--- a/packages/shared-server/src/plugins.ts
+++ b/packages/shared-server/src/plugins.ts
@@ -1,18 +1,42 @@
import { PluginManager } from "@karakeep/shared/plugins";
-let pluginsLoaded = false;
+const pluginLoaderStateKey = "__karakeep_plugins_loader_state__";
+
+const globalPluginLoaderState = globalThis as typeof globalThis & {
+ [pluginLoaderStateKey]?: {
+ loaded: boolean;
+ loading?: Promise<void>;
+ };
+};
+
+const pluginLoaderState = (globalPluginLoaderState[pluginLoaderStateKey] ??= {
+ loaded: false,
+});
+
export async function loadAllPlugins() {
- if (pluginsLoaded) {
+ if (pluginLoaderState.loaded) {
return;
}
- // Load plugins here. Order of plugin loading matter.
- // Queue provider(s)
- await import("@karakeep/plugins/queue-liteque");
- await import("@karakeep/plugins/queue-restate");
- await import("@karakeep/plugins/search-meilisearch");
- // Rate limiters (order matters - last one wins)
- await import("@karakeep/plugins/ratelimit-memory");
- await import("@karakeep/plugins/ratelimit-redis");
- PluginManager.logAllPlugins();
- pluginsLoaded = true;
+ if (pluginLoaderState.loading) {
+ await pluginLoaderState.loading;
+ return;
+ }
+ pluginLoaderState.loading = (async () => {
+ // Load plugins here. Order of plugin loading matter.
+ // Queue provider(s)
+ await import("@karakeep/plugins/queue-liteque");
+ await import("@karakeep/plugins/queue-restate");
+ await import("@karakeep/plugins/search-meilisearch");
+ // Rate limiters (order matters - last one wins)
+ await import("@karakeep/plugins/ratelimit-memory");
+ await import("@karakeep/plugins/ratelimit-redis");
+ PluginManager.logAllPlugins();
+ pluginLoaderState.loaded = true;
+ })();
+
+ try {
+ await pluginLoaderState.loading;
+ } finally {
+ pluginLoaderState.loading = undefined;
+ }
}
diff --git a/packages/shared-server/src/queues.ts b/packages/shared-server/src/queues.ts
index 51fd6d47..30bbff36 100644
--- a/packages/shared-server/src/queues.ts
+++ b/packages/shared-server/src/queues.ts
@@ -9,7 +9,7 @@ import {
} from "@karakeep/shared/queueing";
import { zRuleEngineEventSchema } from "@karakeep/shared/types/rules";
-import { loadAllPlugins } from ".";
+import { loadAllPlugins } from "./plugins";
export enum QueuePriority {
Low = 50,
diff --git a/packages/shared/plugins.ts b/packages/shared/plugins.ts
index 2d03ee39..5ceee91c 100644
--- a/packages/shared/plugins.ts
+++ b/packages/shared/plugins.ts
@@ -30,15 +30,32 @@ export interface PluginProvider<T> {
// Preserve the key-dependent value type: for K, store TPlugin<K>[]
type ProviderMap = { [K in PluginType]: TPlugin<K>[] };
-export class PluginManager {
- private static providers: ProviderMap = {
+const pluginProvidersKey = "__karakeep_plugins_providers__";
+
+function createProviderMap(): ProviderMap {
+ return {
[PluginType.Search]: [],
[PluginType.Queue]: [],
[PluginType.RateLimit]: [],
};
+}
+
+const globalPluginState = globalThis as typeof globalThis & {
+ [pluginProvidersKey]?: ProviderMap;
+};
+
+export class PluginManager {
+ private static providers: ProviderMap = (globalPluginState[
+ pluginProvidersKey
+ ] ??= createProviderMap());
static register<T extends PluginType>(plugin: TPlugin<T>): void {
- PluginManager.providers[plugin.type].push(plugin);
+ const providers = PluginManager.providers[plugin.type];
+ const existingProvider = providers.findIndex((p) => p.name === plugin.name);
+ if (existingProvider >= 0) {
+ return;
+ }
+ providers.push(plugin);
}
static async getClient<T extends PluginType>(
diff --git a/packages/trpc/stats.ts b/packages/trpc/stats.ts
index b252af46..f4d4b19a 100644
--- a/packages/trpc/stats.ts
+++ b/packages/trpc/stats.ts
@@ -1,5 +1,6 @@
import { count, sum } from "drizzle-orm";
import { Counter, Gauge, Histogram, register } from "prom-client";
+import type { Metric } from "prom-client";
import { db } from "@karakeep/db";
import { assets, bookmarks, subscriptions, users } from "@karakeep/db/schema";
@@ -18,166 +19,206 @@ import {
} from "@karakeep/shared-server";
import serverConfig from "@karakeep/shared/config";
+function getOrCreateMetric<T extends Metric>(
+ name: string,
+ createMetric: () => T,
+): T {
+ return (register.getSingleMetric(name) as T | undefined) ?? createMetric();
+}
+
// Queue metrics
-const queuePendingJobsGauge = new Gauge({
- name: "karakeep_queue_jobs",
- help: "Number of jobs in each background queue",
- labelNames: ["queue_name", "status"],
- async collect() {
- const queues = [
- { name: "link_crawler", queue: LinkCrawlerQueue },
- { name: "low_priority_crawler", queue: LowPriorityCrawlerQueue },
- { name: "backup", queue: BackupQueue },
- { name: "openai", queue: OpenAIQueue },
- { name: "search_indexing", queue: SearchIndexingQueue },
- { name: "admin_maintenance", queue: AdminMaintenanceQueue },
- { name: "video_worker", queue: VideoWorkerQueue },
- { name: "feed", queue: FeedQueue },
- { name: "asset_preprocessing", queue: AssetPreprocessingQueue },
- { name: "webhook", queue: WebhookQueue },
- { name: "rule_engine", queue: RuleEngineQueue },
- ];
+const queuePendingJobsGauge = getOrCreateMetric(
+ "karakeep_queue_jobs",
+ () =>
+ new Gauge({
+ name: "karakeep_queue_jobs",
+ help: "Number of jobs in each background queue",
+ labelNames: ["queue_name", "status"],
+ async collect() {
+ const queues = [
+ { name: "link_crawler", queue: LinkCrawlerQueue },
+ { name: "low_priority_crawler", queue: LowPriorityCrawlerQueue },
+ { name: "backup", queue: BackupQueue },
+ { name: "openai", queue: OpenAIQueue },
+ { name: "search_indexing", queue: SearchIndexingQueue },
+ { name: "admin_maintenance", queue: AdminMaintenanceQueue },
+ { name: "video_worker", queue: VideoWorkerQueue },
+ { name: "feed", queue: FeedQueue },
+ { name: "asset_preprocessing", queue: AssetPreprocessingQueue },
+ { name: "webhook", queue: WebhookQueue },
+ { name: "rule_engine", queue: RuleEngineQueue },
+ ];
- const stats = await Promise.all(
- queues.map(async ({ name, queue }) => {
- try {
- return {
- ...(await queue.stats()),
- name,
- };
- } catch (error) {
- console.error(`Failed to get stats for queue ${name}:`, error);
- return { name, pending: 0, pending_retry: 0, failed: 0, running: 0 };
- }
- }),
- );
+ const stats = await Promise.all(
+ queues.map(async ({ name, queue }) => {
+ try {
+ return {
+ ...(await queue.stats()),
+ name,
+ };
+ } catch (error) {
+ console.error(`Failed to get stats for queue ${name}:`, error);
+ return {
+ name,
+ pending: 0,
+ pending_retry: 0,
+ failed: 0,
+ running: 0,
+ };
+ }
+ }),
+ );
- stats.forEach(({ name, pending, pending_retry, failed, running }) => {
- this.set({ queue_name: name, status: "pending" }, pending);
- this.set({ queue_name: name, status: "pending_retry" }, pending_retry);
- this.set({ queue_name: name, status: "failed" }, failed);
- this.set({ queue_name: name, status: "running" }, running);
- });
- },
-});
+ stats.forEach(({ name, pending, pending_retry, failed, running }) => {
+ this.set({ queue_name: name, status: "pending" }, pending);
+ this.set(
+ { queue_name: name, status: "pending_retry" },
+ pending_retry,
+ );
+ this.set({ queue_name: name, status: "failed" }, failed);
+ this.set({ queue_name: name, status: "running" }, running);
+ });
+ },
+ }),
+);
// User metrics
-const totalUsersGauge = new Gauge({
- name: "karakeep_total_users",
- help: "Total number of users in the system",
- async collect() {
- try {
- const result = await db.select({ count: count() }).from(users);
- this.set(result[0]?.count ?? 0);
- } catch (error) {
- console.error("Failed to get user count:", error);
- this.set(0);
- }
- },
-});
-
-const subscriptionStatusGauge = serverConfig.stripe.isConfigured
- ? new Gauge({
- name: "karakeep_subscription_status",
- help: "Total number of subscriptions per status",
- labelNames: ["status", "tier"],
+const totalUsersGauge = getOrCreateMetric(
+ "karakeep_total_users",
+ () =>
+ new Gauge({
+ name: "karakeep_total_users",
+ help: "Total number of users in the system",
async collect() {
- this.reset();
try {
- const results = await db
- .select({
- status: subscriptions.status,
- tier: subscriptions.tier,
- count: count(),
- })
- .from(subscriptions)
- .groupBy(subscriptions.status, subscriptions.tier);
- for (const result of results) {
- this.set(
- { status: result.status, tier: result.tier },
- result.count,
- );
- }
+ const result = await db.select({ count: count() }).from(users);
+ this.set(result[0]?.count ?? 0);
} catch (error) {
- console.error("Failed to get subscription status:", error);
+ console.error("Failed to get user count:", error);
+ this.set(0);
}
},
- })
- : null;
+ }),
+);
+
+if (serverConfig.stripe.isConfigured) {
+ getOrCreateMetric(
+ "karakeep_subscription_status",
+ () =>
+ new Gauge({
+ name: "karakeep_subscription_status",
+ help: "Total number of subscriptions per status",
+ labelNames: ["status", "tier"],
+ async collect() {
+ this.reset();
+ try {
+ const results = await db
+ .select({
+ status: subscriptions.status,
+ tier: subscriptions.tier,
+ count: count(),
+ })
+ .from(subscriptions)
+ .groupBy(subscriptions.status, subscriptions.tier);
+ for (const result of results) {
+ this.set(
+ { status: result.status, tier: result.tier },
+ result.count,
+ );
+ }
+ } catch (error) {
+ console.error("Failed to get subscription status:", error);
+ }
+ },
+ }),
+ );
+}
// Asset metrics
-const totalAssetSizeGauge = new Gauge({
- name: "karakeep_total_asset_size_bytes",
- help: "Total size of all assets in bytes",
- async collect() {
- try {
- const result = await db
- .select({ totalSize: sum(assets.size) })
- .from(assets);
- this.set(Number(result[0]?.totalSize ?? 0));
- } catch (error) {
- console.error("Failed to get total asset size:", error);
- this.set(0);
- }
- },
-});
+const totalAssetSizeGauge = getOrCreateMetric(
+ "karakeep_total_asset_size_bytes",
+ () =>
+ new Gauge({
+ name: "karakeep_total_asset_size_bytes",
+ help: "Total size of all assets in bytes",
+ async collect() {
+ try {
+ const result = await db
+ .select({ totalSize: sum(assets.size) })
+ .from(assets);
+ this.set(Number(result[0]?.totalSize ?? 0));
+ } catch (error) {
+ console.error("Failed to get total asset size:", error);
+ this.set(0);
+ }
+ },
+ }),
+);
// Bookmark metrics
-const totalBookmarksGauge = new Gauge({
- name: "karakeep_total_bookmarks",
- help: "Total number of bookmarks in the system",
- async collect() {
- try {
- const result = await db.select({ count: count() }).from(bookmarks);
- this.set(result[0]?.count ?? 0);
- } catch (error) {
- console.error("Failed to get bookmark count:", error);
- this.set(0);
- }
- },
-});
+const totalBookmarksGauge = getOrCreateMetric(
+ "karakeep_total_bookmarks",
+ () =>
+ new Gauge({
+ name: "karakeep_total_bookmarks",
+ help: "Total number of bookmarks in the system",
+ async collect() {
+ try {
+ const result = await db.select({ count: count() }).from(bookmarks);
+ this.set(result[0]?.count ?? 0);
+ } catch (error) {
+ console.error("Failed to get bookmark count:", error);
+ this.set(0);
+ }
+ },
+ }),
+);
// Bookmark creation metrics
-const bookmarkCreationCounter = new Counter({
- name: "karakeep_bookmark_creations_total",
- help: "Total number of bookmarks created",
- labelNames: ["source"],
-});
+const bookmarkCreationCounter = getOrCreateMetric(
+ "karakeep_bookmark_creations_total",
+ () =>
+ new Counter({
+ name: "karakeep_bookmark_creations_total",
+ help: "Total number of bookmarks created",
+ labelNames: ["source"],
+ }),
+);
// Api metrics
-const apiRequestsTotalCounter = new Counter({
- name: "karakeep_trpc_requests_total",
- help: "Total number of API requests",
- labelNames: ["type", "path", "is_error"],
-});
-
-const apiErrorsTotalCounter = new Counter({
- name: "karakeep_trpc_errors_total",
- help: "Total number of API requests",
- labelNames: ["type", "path", "code"],
-});
+const apiRequestsTotalCounter = getOrCreateMetric(
+ "karakeep_trpc_requests_total",
+ () =>
+ new Counter({
+ name: "karakeep_trpc_requests_total",
+ help: "Total number of API requests",
+ labelNames: ["type", "path", "is_error"],
+ }),
+);
-const apiRequestDurationSummary = new Histogram({
- name: "karakeep_trpc_request_duration_seconds",
- help: "Duration of tRPC requests in seconds",
- labelNames: ["type", "path"],
- buckets: [
- 5e-3, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10,
- ],
-});
+const apiErrorsTotalCounter = getOrCreateMetric(
+ "karakeep_trpc_errors_total",
+ () =>
+ new Counter({
+ name: "karakeep_trpc_errors_total",
+ help: "Total number of API requests",
+ labelNames: ["type", "path", "code"],
+ }),
+);
-// Register all metrics
-register.registerMetric(queuePendingJobsGauge);
-register.registerMetric(totalUsersGauge);
-if (subscriptionStatusGauge) {
- register.registerMetric(subscriptionStatusGauge);
-}
-register.registerMetric(totalAssetSizeGauge);
-register.registerMetric(totalBookmarksGauge);
-register.registerMetric(bookmarkCreationCounter);
-register.registerMetric(apiRequestsTotalCounter);
-register.registerMetric(apiErrorsTotalCounter);
-register.registerMetric(apiRequestDurationSummary);
+const apiRequestDurationSummary = getOrCreateMetric(
+ "karakeep_trpc_request_duration_seconds",
+ () =>
+ new Histogram({
+ name: "karakeep_trpc_request_duration_seconds",
+ help: "Duration of tRPC requests in seconds",
+ labelNames: ["type", "path"],
+ buckets: [
+ 5e-3, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5,
+ 10,
+ ],
+ }),
+);
export {
queuePendingJobsGauge,