Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

Add on-demand browser lifecycle mode (BROWSER_LIFECYCLE=on-demand)

Profiteroles, LLCd32fb58df1fa
diff --git a/.env.sample b/.env.sample
index fc71395a..e2608ae4 100644
--- a/.env.sample
+++ b/.env.sample
@@ -18,6 +18,18 @@ DISABLE_NEW_SSO_USERS=
MAX_LINKS_PER_USER=
ARCHIVE_TAKE_COUNT=
BROWSER_TIMEOUT=
+
+# Browser lifecycle mode. "persistent" (default) keeps Chromium running at all
+# times with a 30-minute restart rotation. "on-demand" launches Chromium only
+# when links need processing and shuts it down after an idle period, freeing
+# 300-800 MB of RAM on instances with sporadic usage.
+# BROWSER_LIFECYCLE=persistent
+
+# How long (ms) to keep the on-demand browser alive after the last processed
+# link. Prevents rapid launch/close cycles when saving multiple links in quick
+# succession. Only used when BROWSER_LIFECYCLE=on-demand. Default: 60000 (1 min).
+# BROWSER_IDLE_TIMEOUT_MS=60000
+
IGNORE_URL_SIZE_LIMIT=
NEXT_PUBLIC_DEMO=
NEXT_PUBLIC_DEMO_USERNAME=
diff --git a/apps/worker/workers/linkProcessing.ts b/apps/worker/workers/linkProcessing.ts
index 886c3b4f..5ac0e215 100644
--- a/apps/worker/workers/linkProcessing.ts
+++ b/apps/worker/workers/linkProcessing.ts
@@ -4,17 +4,36 @@ import { delay } from "@linkwarden/lib/utils";
import getLinkBatchFairly from "../lib/getLinkBatchFairly";
import { launchBrowser } from "../lib/browser";
import { countUnprocessedBillableLinks } from "../lib/countUnprocessedBillableLinks";
+import { Browser } from "playwright";
const ARCHIVE_TAKE_COUNT = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;
const BROWSER_MAX_AGE_MS = 30 * 60 * 1000; // 30 minutes
+// On-demand mode: launch browser only when links need processing, close when
+// idle. Dramatically reduces memory usage on instances with sporadic traffic.
+// Default ("persistent") preserves existing always-on behavior.
+const BROWSER_LIFECYCLE = (
+ process.env.BROWSER_LIFECYCLE || "persistent"
+).toLowerCase();
+const BROWSER_IDLE_TIMEOUT_MS =
+ Number(process.env.BROWSER_IDLE_TIMEOUT_MS || "") || 60_000; // 1 minute
+
export async function linkProcessing(interval = 10) {
console.log("\x1b[34m%s\x1b[0m", "Starting link processing...");
+ if (BROWSER_LIFECYCLE === "on-demand") {
+ await linkProcessingOnDemand(interval);
+ } else {
+ await linkProcessingPersistent(interval);
+ }
+}
+
+// ── Persistent mode (existing behavior) ─────────────────────────────────────
+
+async function linkProcessingPersistent(interval: number) {
let browser = await launchBrowser();
let browserStartTs = Date.now();
- // Helper to (re)launch browser
const restartBrowser = async (reason: string) => {
try {
if (browser && browser.isConnected()) {
@@ -27,7 +46,6 @@ export async function linkProcessing(interval = 10) {
};
while (true) {
- // Restart every 30 minutes to prevent clogging
if (Date.now() - browserStartTs >= BROWSER_MAX_AGE_MS) {
await restartBrowser("30-minute rotation");
}
@@ -82,3 +100,124 @@ export async function linkProcessing(interval = 10) {
await delay(interval);
}
}
+
+// ── On-demand mode ──────────────────────────────────────────────────────────
+//
+// Browser lifecycle:
+// idle (no browser) → links found → launch browser → process all queued
+// links → start idle timer → if no new links within BROWSER_IDLE_TIMEOUT_MS
+// → close browser → back to idle
+//
+// This avoids launching/closing Chromium for every single link while still
+// freeing ~300-800 MB of RAM when the instance is idle.
+
+async function linkProcessingOnDemand(interval: number) {
+ let browser: Browser | null = null;
+ let browserStartTs = 0;
+ let lastActivityTs = 0;
+
+ const ensureBrowser = async (): Promise<Browser> => {
+ if (browser && browser.isConnected()) {
+ // Rotate if the browser has been alive too long (same leak prevention
+ // as persistent mode).
+ if (Date.now() - browserStartTs >= BROWSER_MAX_AGE_MS) {
+ console.log(
+ "\x1b[34m%s\x1b[0m",
+ "Rotating on-demand browser (30-minute max age)..."
+ );
+ await closeBrowser();
+ } else {
+ return browser;
+ }
+ }
+
+ console.log("\x1b[34m%s\x1b[0m", "Launching browser on demand...");
+ browser = await launchBrowser();
+ browserStartTs = Date.now();
+ return browser;
+ };
+
+ const closeBrowser = async () => {
+ if (!browser) return;
+ try {
+ if (browser.isConnected()) {
+ await browser.close();
+ }
+ } catch {}
+ browser = null;
+ browserStartTs = 0;
+ console.log(
+ "\x1b[34m%s\x1b[0m",
+ "Browser closed (on-demand idle shutdown)."
+ );
+ };
+
+ while (true) {
+ const links = await getLinkBatchFairly({
+ maxBatchLinks: ARCHIVE_TAKE_COUNT,
+ });
+
+ if (links.length === 0) {
+ // No work — close the browser if idle timeout has elapsed.
+ if (
+ browser &&
+ lastActivityTs > 0 &&
+ Date.now() - lastActivityTs >= BROWSER_IDLE_TIMEOUT_MS
+ ) {
+ await closeBrowser();
+ }
+
+ await delay(interval);
+ continue;
+ }
+
+ // Work available — ensure browser is running.
+ const activeBrowser = await ensureBrowser();
+ lastActivityTs = Date.now();
+
+ const archiveLink = async (link: LinkWithCollectionOwnerAndTags) => {
+ try {
+ console.log(
+ "\x1b[34m%s\x1b[0m",
+ `- Link ${link.url} for user ${link.collection.ownerId}`
+ );
+
+ await archiveHandler(link, activeBrowser);
+
+ console.log(
+ "\x1b[34m%s\x1b[0m",
+ `Succeeded processing link ${link.url} for user ${link.collection.ownerId}.`
+ );
+ } catch (error: any) {
+ console.error(
+ "\x1b[34m%s\x1b[0m",
+ `Error processing link ${link.url} for user ${link.collection.ownerId}:`,
+ error
+ );
+
+ if (!activeBrowser.isConnected?.()) {
+ console.log(
+ "\x1b[34m%s\x1b[0m",
+ "Browser disconnected during processing, will relaunch on next batch."
+ );
+ browser = null;
+ browserStartTs = 0;
+ }
+ }
+ };
+
+ const processingPromises = links.map((e) => archiveLink(e));
+ await Promise.allSettled(processingPromises);
+
+ const unprocessedLinkCount = await countUnprocessedBillableLinks();
+
+ console.log(
+ "\x1b[34m%s\x1b[0m",
+ `Processed ${links.length} link${
+ links.length === 1 ? "" : "s"
+ }, ${unprocessedLinkCount} left.`
+ );
+
+ await delay(interval);
+ }
+}