Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

use a single browser with separate context

daniel31x13577f8600ae49
diff --git a/apps/worker/lib/archiveHandler.ts b/apps/worker/lib/archiveHandler.ts
index 8ae19e2c..dc6a5a60 100644
--- a/apps/worker/lib/archiveHandler.ts
+++ b/apps/worker/lib/archiveHandler.ts
@@ -2,8 +2,6 @@ import {
Browser,
BrowserContext,
BrowserContextOptions,
- LaunchOptions,
- chromium,
devices,
} from "playwright";
import { prisma } from "@linkwarden/prisma";
@@ -21,11 +19,13 @@ import autoTagLink from "./autoTagLink";
import { LinkWithCollectionOwnerAndTags } from "@linkwarden/types";
import { isArchivalTag } from "@linkwarden/lib";
import { ArchivalSettings } from "@linkwarden/types";
+import { getDefaultContextOptions } from "./browser";
const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;
export default async function archiveHandler(
- link: LinkWithCollectionOwnerAndTags
+ link: LinkWithCollectionOwnerAndTags,
+ browser: Browser
) {
const user = link.collection?.owner;
@@ -56,17 +56,14 @@ export default async function archiveHandler(
: undefined,
},
});
-
return;
}
const abortController = new AbortController();
-
const timeoutPromise = new Promise((_, reject) => {
setTimeout(() => {
abortController.abort();
-
- return reject(
+ reject(
new Error(
`Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`
)
@@ -74,14 +71,14 @@ export default async function archiveHandler(
}, BROWSER_TIMEOUT * 60000);
});
- const { browser, context } = await getBrowser();
+ const contextOptions = getDefaultContextOptions();
+ const context = await browser.newContext(contextOptions);
const page = await context.newPage();
createFolder({ filePath: `archives/preview/${link.collectionId}` });
createFolder({ filePath: `archives/${link.collectionId}` });
const archivalTags = link.tags.filter(isArchivalTag);
-
const archivalSettings: ArchivalSettings =
archivalTags.length > 0
? {
@@ -114,18 +111,17 @@ export default async function archiveHandler(
);
// send to archive.org
- if (archivalSettings.archiveAsWaybackMachine && link.url)
+ if (archivalSettings.archiveAsWaybackMachine && link.url) {
sendToWayback(link.url);
+ }
if (linkType === "image" && !link.image) {
- await imageHandler(link, imageExtension); // archive image (jpeg/png)
+ await imageHandler(link, imageExtension);
return;
} else if (linkType === "pdf" && !link.pdf) {
- await pdfHandler(link); // archive pdf
+ await pdfHandler(link);
return;
} else if (link.url) {
- // archive url
-
await page.goto(link.url, { waitUntil: "domcontentloaded" });
const metaDescription = await page.evaluate(() => {
@@ -148,10 +144,9 @@ export default async function archiveHandler(
if (
(archivalSettings.archiveAsScreenshot && !link.image) ||
(archivalSettings.archiveAsPDF && !link.pdf)
- )
+ ) {
await handleScreenshotAndPdf(link, page, archivalSettings);
-
- await browser.close();
+ }
// Auto-tagging
if (
@@ -163,16 +158,22 @@ export default async function archiveHandler(
process.env.AZURE_API_KEY ||
process.env.ANTHROPIC_API_KEY ||
process.env.OPENROUTER_API_KEY)
- )
+ ) {
await autoTagLink(user, link.id, metaDescription);
+ }
// Monolith
- if (archivalSettings.archiveAsMonolith && !link.monolith && link.url)
+ if (
+ archivalSettings.archiveAsMonolith &&
+ !link.monolith &&
+ link.url
+ ) {
await handleMonolith(link, content, abortController.signal).catch(
(err) => {
console.error(err);
}
);
+ }
}
})(),
timeoutPromise,
@@ -186,7 +187,7 @@ export default async function archiveHandler(
where: { id: link.id },
});
- if (finalLink)
+ if (finalLink) {
await prisma.link.update({
where: { id: link.id },
data: {
@@ -203,13 +204,11 @@ export default async function archiveHandler(
: undefined,
},
});
- else {
+ } else {
await removeFiles(link.id, link.collectionId);
}
- if (browser && browser.isConnected()) {
- await browser.close();
- }
+ await context?.close().catch(() => {});
}
}
@@ -239,60 +238,8 @@ async function determineLinkType(
await prisma.link.update({
where: { id: linkId },
- data: {
- type: linkType,
- },
+ data: { type: linkType },
});
return { linkType, imageExtension };
}
-
-// Construct browser launch options based on environment variables.
-export function getBrowserOptions(): LaunchOptions {
- let browserOptions: LaunchOptions = {};
-
- if (process.env.PROXY) {
- browserOptions.proxy = {
- server: process.env.PROXY,
- bypass: process.env.PROXY_BYPASS,
- username: process.env.PROXY_USERNAME,
- password: process.env.PROXY_PASSWORD,
- };
- }
-
- if (
- process.env.PLAYWRIGHT_LAUNCH_OPTIONS_EXECUTABLE_PATH &&
- !process.env.PLAYWRIGHT_WS_URL
- ) {
- browserOptions.executablePath =
- process.env.PLAYWRIGHT_LAUNCH_OPTIONS_EXECUTABLE_PATH;
- }
-
- return browserOptions;
-}
-
-async function getBrowser(): Promise<{
- browser: Browser;
- context: BrowserContext;
-}> {
- const browserOptions = getBrowserOptions();
- let browser: Browser;
- let contextOptions: BrowserContextOptions = {
- ...devices["Desktop Chrome"],
- ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",
- };
-
- if (process.env.PLAYWRIGHT_WS_URL) {
- browser = await chromium.connectOverCDP(process.env.PLAYWRIGHT_WS_URL);
- contextOptions = {
- ...contextOptions,
- ...browserOptions,
- };
- } else {
- browser = await chromium.launch(browserOptions);
- }
-
- const context = await browser.newContext(contextOptions);
-
- return { browser, context };
-}
diff --git a/apps/worker/lib/browser.ts b/apps/worker/lib/browser.ts
new file mode 100644
index 00000000..203b67f9
--- /dev/null
+++ b/apps/worker/lib/browser.ts
@@ -0,0 +1,59 @@
+import {
+ chromium,
+ devices,
+ type Browser,
+ type LaunchOptions,
+ type BrowserContextOptions,
+} from "playwright";
+
+export function getBrowserOptions(): LaunchOptions {
+ let browserOptions: LaunchOptions = {
+ // headless: false,
+ };
+
+ if (process.env.PROXY) {
+ browserOptions.proxy = {
+ server: process.env.PROXY,
+ bypass: process.env.PROXY_BYPASS,
+ username: process.env.PROXY_USERNAME,
+ password: process.env.PROXY_PASSWORD,
+ };
+ }
+
+ if (
+ process.env.PLAYWRIGHT_LAUNCH_OPTIONS_EXECUTABLE_PATH &&
+ !process.env.PLAYWRIGHT_WS_URL
+ ) {
+ browserOptions.executablePath =
+ process.env.PLAYWRIGHT_LAUNCH_OPTIONS_EXECUTABLE_PATH;
+ }
+
+ return browserOptions;
+}
+
+export function getDefaultContextOptions(): BrowserContextOptions {
+ const base: BrowserContextOptions = {
+ ...devices["Desktop Chrome"],
+ ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",
+ };
+
+ if (process.env.PLAYWRIGHT_WS_URL) {
+ const launchLike = getBrowserOptions();
+ return {
+ ...base,
+ ...(launchLike as unknown as Partial<BrowserContextOptions>),
+ };
+ }
+
+ return base;
+}
+
+export async function launchBrowser(): Promise<Browser> {
+ const browserOptions = getBrowserOptions();
+
+ if (process.env.PLAYWRIGHT_WS_URL) {
+ return chromium.connectOverCDP(process.env.PLAYWRIGHT_WS_URL);
+ }
+
+ return chromium.launch(browserOptions);
+}
diff --git a/apps/worker/worker.ts b/apps/worker/worker.ts
index 803b704d..70fc5092 100644
--- a/apps/worker/worker.ts
+++ b/apps/worker/worker.ts
@@ -1,5 +1,5 @@
import { startIndexing } from "./workers/linkIndexing";
-import { startProcessing } from "./workers/linkProcessing";
+import { linkProcessing } from "./workers/linkProcessing";
import { startRSSPolling } from "./workers/rssPolling";
const workerIntervalInSeconds =
@@ -8,7 +8,7 @@ const workerIntervalInSeconds =
async function init() {
console.log("\x1b[34m%s\x1b[0m", "Initializing the worker...");
startRSSPolling();
- startProcessing(workerIntervalInSeconds);
+ linkProcessing(workerIntervalInSeconds);
startIndexing(workerIntervalInSeconds);
}
diff --git a/apps/worker/workers/linkProcessing.ts b/apps/worker/workers/linkProcessing.ts
index 97881c31..b6e5eea4 100644
--- a/apps/worker/workers/linkProcessing.ts
+++ b/apps/worker/workers/linkProcessing.ts
@@ -3,12 +3,35 @@ import archiveHandler from "../lib/archiveHandler";
import { LinkWithCollectionOwnerAndTags } from "@linkwarden/types";
import { delay } from "@linkwarden/lib";
import getLinkBatchFairly from "../lib/getLinkBatchFairly";
+import { launchBrowser } from "../lib/browser";
const ARCHIVE_TAKE_COUNT = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;
+const BROWSER_MAX_AGE_MS = 30 * 60 * 1000; // 30 minutes
-export async function startProcessing(interval = 10) {
+export async function linkProcessing(interval = 10) {
console.log("\x1b[34m%s\x1b[0m", "Starting link processing...");
+
+ let browser = await launchBrowser();
+ let browserStartTs = Date.now();
+
+ // Helper to (re)launch browser
+ const restartBrowser = async (reason: string) => {
+ try {
+ if (browser && browser.isConnected()) {
+ await browser.close();
+ }
+ } catch {}
+ console.log("\x1b[34m%s\x1b[0m", `Restarting main browser (${reason})...`);
+ browser = await launchBrowser();
+ browserStartTs = Date.now();
+ };
+
while (true) {
+ // Restart every 30 minutes to prevent clogging
+ if (Date.now() - browserStartTs >= BROWSER_MAX_AGE_MS) {
+ await restartBrowser("30-minute rotation");
+ }
+
const links = await getLinkBatchFairly({
maxBatchLinks: ARCHIVE_TAKE_COUNT,
});
@@ -25,31 +48,30 @@ export async function startProcessing(interval = 10) {
`- Link ${link.url} for user ${link.collection.ownerId}`
);
- await archiveHandler(link);
+ await archiveHandler(link, browser);
console.log(
"\x1b[34m%s\x1b[0m",
`Succeeded processing link ${link.url} for user ${link.collection.ownerId}.`
);
- } catch (error) {
+ } catch (error: any) {
console.error(
"\x1b[34m%s\x1b[0m",
`Error processing link ${link.url} for user ${link.collection.ownerId}:`,
error
);
+
+ if (!browser.isConnected?.()) {
+ await restartBrowser("browser disconnected");
+ }
}
};
- // Process each link in the batch concurrently
const processingPromises = links.map((e) => archiveLink(e));
-
await Promise.allSettled(processingPromises);
const unprocessedLinkCount = await prisma.link.count({
- where: {
- lastPreserved: null,
- url: { not: null },
- },
+ where: { lastPreserved: null, url: { not: null } },
});
console.log(