⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
use a single browser with separate context
daniel31x13577f8600ae49
diff --git a/apps/worker/lib/archiveHandler.ts b/apps/worker/lib/archiveHandler.tsindex 8ae19e2c..dc6a5a60 100644--- a/apps/worker/lib/archiveHandler.ts+++ b/apps/worker/lib/archiveHandler.ts@@ -2,8 +2,6 @@ import {Browser,BrowserContext,BrowserContextOptions,- LaunchOptions,- chromium,devices,} from "playwright";import { prisma } from "@linkwarden/prisma";@@ -21,11 +19,13 @@ import autoTagLink from "./autoTagLink";import { LinkWithCollectionOwnerAndTags } from "@linkwarden/types";import { isArchivalTag } from "@linkwarden/lib";import { ArchivalSettings } from "@linkwarden/types";+import { getDefaultContextOptions } from "./browser";const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;export default async function archiveHandler(- link: LinkWithCollectionOwnerAndTags+ link: LinkWithCollectionOwnerAndTags,+ browser: Browser) {const user = link.collection?.owner;@@ -56,17 +56,14 @@ export default async function archiveHandler(: undefined,},});-return;}const abortController = new AbortController();-const timeoutPromise = new Promise((_, reject) => {setTimeout(() => {abortController.abort();-- return reject(+ reject(new Error(`Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`)@@ -74,14 +71,14 @@ export default async function archiveHandler(}, BROWSER_TIMEOUT * 60000);});- const { browser, context } = await getBrowser();+ const contextOptions = getDefaultContextOptions();+ const context = await browser.newContext(contextOptions);const page = await context.newPage();createFolder({ filePath: `archives/preview/${link.collectionId}` });createFolder({ filePath: `archives/${link.collectionId}` });const archivalTags = link.tags.filter(isArchivalTag);-const archivalSettings: ArchivalSettings =archivalTags.length > 0? {@@ -114,18 +111,17 @@ export default async function archiveHandler();// send to archive.org- if (archivalSettings.archiveAsWaybackMachine && link.url)+ if (archivalSettings.archiveAsWaybackMachine && link.url) {sendToWayback(link.url);+ }if (linkType === "image" && !link.image) {- await imageHandler(link, imageExtension); // archive image (jpeg/png)+ await imageHandler(link, imageExtension);return;} else if (linkType === "pdf" && !link.pdf) {- await pdfHandler(link); // archive pdf+ await pdfHandler(link);return;} else if (link.url) {- // archive url-await page.goto(link.url, { waitUntil: "domcontentloaded" });const metaDescription = await page.evaluate(() => {@@ -148,10 +144,9 @@ export default async function archiveHandler(if ((archivalSettings.archiveAsScreenshot && !link.image) ||(archivalSettings.archiveAsPDF && !link.pdf)- )+ ) {await handleScreenshotAndPdf(link, page, archivalSettings);-- await browser.close();+ }// Auto-taggingif (@@ -163,16 +158,22 @@ export default async function archiveHandler(process.env.AZURE_API_KEY ||process.env.ANTHROPIC_API_KEY ||process.env.OPENROUTER_API_KEY)- )+ ) {await autoTagLink(user, link.id, metaDescription);+ }// Monolith- if (archivalSettings.archiveAsMonolith && !link.monolith && link.url)+ if (+ archivalSettings.archiveAsMonolith &&+ !link.monolith &&+ link.url+ ) {await handleMonolith(link, content, abortController.signal).catch((err) => {console.error(err);});+ }}})(),timeoutPromise,@@ -186,7 +187,7 @@ export default async function archiveHandler(where: { id: link.id },});- if (finalLink)+ if (finalLink) {await prisma.link.update({where: { id: link.id },data: {@@ -203,13 +204,11 @@ export default async function archiveHandler(: undefined,},});- else {+ } else {await removeFiles(link.id, link.collectionId);}- if (browser && browser.isConnected()) {- await browser.close();- }+ await context?.close().catch(() => {});}}@@ -239,60 +238,8 @@ async function determineLinkType(await prisma.link.update({where: { id: linkId },- data: {- type: linkType,- },+ data: { type: linkType },});return { linkType, imageExtension };}--// Construct browser launch options based on environment variables.-export function getBrowserOptions(): LaunchOptions {- let browserOptions: LaunchOptions = {};-- if (process.env.PROXY) {- browserOptions.proxy = {- server: process.env.PROXY,- bypass: process.env.PROXY_BYPASS,- username: process.env.PROXY_USERNAME,- password: process.env.PROXY_PASSWORD,- };- }-- if (- process.env.PLAYWRIGHT_LAUNCH_OPTIONS_EXECUTABLE_PATH &&- !process.env.PLAYWRIGHT_WS_URL- ) {- browserOptions.executablePath =- process.env.PLAYWRIGHT_LAUNCH_OPTIONS_EXECUTABLE_PATH;- }-- return browserOptions;-}--async function getBrowser(): Promise<{- browser: Browser;- context: BrowserContext;-}> {- const browserOptions = getBrowserOptions();- let browser: Browser;- let contextOptions: BrowserContextOptions = {- ...devices["Desktop Chrome"],- ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",- };-- if (process.env.PLAYWRIGHT_WS_URL) {- browser = await chromium.connectOverCDP(process.env.PLAYWRIGHT_WS_URL);- contextOptions = {- ...contextOptions,- ...browserOptions,- };- } else {- browser = await chromium.launch(browserOptions);- }-- const context = await browser.newContext(contextOptions);-- return { browser, context };-}diff --git a/apps/worker/lib/browser.ts b/apps/worker/lib/browser.tsnew file mode 100644index 00000000..203b67f9--- /dev/null+++ b/apps/worker/lib/browser.ts@@ -0,0 +1,59 @@+import {+ chromium,+ devices,+ type Browser,+ type LaunchOptions,+ type BrowserContextOptions,+} from "playwright";++export function getBrowserOptions(): LaunchOptions {+ let browserOptions: LaunchOptions = {+ // headless: false,+ };++ if (process.env.PROXY) {+ browserOptions.proxy = {+ server: process.env.PROXY,+ bypass: process.env.PROXY_BYPASS,+ username: process.env.PROXY_USERNAME,+ password: process.env.PROXY_PASSWORD,+ };+ }++ if (+ process.env.PLAYWRIGHT_LAUNCH_OPTIONS_EXECUTABLE_PATH &&+ !process.env.PLAYWRIGHT_WS_URL+ ) {+ browserOptions.executablePath =+ process.env.PLAYWRIGHT_LAUNCH_OPTIONS_EXECUTABLE_PATH;+ }++ return browserOptions;+}++export function getDefaultContextOptions(): BrowserContextOptions {+ const base: BrowserContextOptions = {+ ...devices["Desktop Chrome"],+ ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",+ };++ if (process.env.PLAYWRIGHT_WS_URL) {+ const launchLike = getBrowserOptions();+ return {+ ...base,+ ...(launchLike as unknown as Partial<BrowserContextOptions>),+ };+ }++ return base;+}++export async function launchBrowser(): Promise<Browser> {+ const browserOptions = getBrowserOptions();++ if (process.env.PLAYWRIGHT_WS_URL) {+ return chromium.connectOverCDP(process.env.PLAYWRIGHT_WS_URL);+ }++ return chromium.launch(browserOptions);+}diff --git a/apps/worker/worker.ts b/apps/worker/worker.tsindex 803b704d..70fc5092 100644--- a/apps/worker/worker.ts+++ b/apps/worker/worker.ts@@ -1,5 +1,5 @@import { startIndexing } from "./workers/linkIndexing";-import { startProcessing } from "./workers/linkProcessing";+import { linkProcessing } from "./workers/linkProcessing";import { startRSSPolling } from "./workers/rssPolling";const workerIntervalInSeconds =@@ -8,7 +8,7 @@ const workerIntervalInSeconds =async function init() {console.log("\x1b[34m%s\x1b[0m", "Initializing the worker...");startRSSPolling();- startProcessing(workerIntervalInSeconds);+ linkProcessing(workerIntervalInSeconds);startIndexing(workerIntervalInSeconds);}diff --git a/apps/worker/workers/linkProcessing.ts b/apps/worker/workers/linkProcessing.tsindex 97881c31..b6e5eea4 100644--- a/apps/worker/workers/linkProcessing.ts+++ b/apps/worker/workers/linkProcessing.ts@@ -3,12 +3,35 @@ import archiveHandler from "../lib/archiveHandler";import { LinkWithCollectionOwnerAndTags } from "@linkwarden/types";import { delay } from "@linkwarden/lib";import getLinkBatchFairly from "../lib/getLinkBatchFairly";+import { launchBrowser } from "../lib/browser";const ARCHIVE_TAKE_COUNT = Number(process.env.ARCHIVE_TAKE_COUNT || "") || 5;+const BROWSER_MAX_AGE_MS = 30 * 60 * 1000; // 30 minutes-export async function startProcessing(interval = 10) {+export async function linkProcessing(interval = 10) {console.log("\x1b[34m%s\x1b[0m", "Starting link processing...");++ let browser = await launchBrowser();+ let browserStartTs = Date.now();++ // Helper to (re)launch browser+ const restartBrowser = async (reason: string) => {+ try {+ if (browser && browser.isConnected()) {+ await browser.close();+ }+ } catch {}+ console.log("\x1b[34m%s\x1b[0m", `Restarting main browser (${reason})...`);+ browser = await launchBrowser();+ browserStartTs = Date.now();+ };+while (true) {+ // Restart every 30 minutes to prevent clogging+ if (Date.now() - browserStartTs >= BROWSER_MAX_AGE_MS) {+ await restartBrowser("30-minute rotation");+ }+const links = await getLinkBatchFairly({maxBatchLinks: ARCHIVE_TAKE_COUNT,});@@ -25,31 +48,30 @@ export async function startProcessing(interval = 10) {`- Link ${link.url} for user ${link.collection.ownerId}`);- await archiveHandler(link);+ await archiveHandler(link, browser);console.log("\x1b[34m%s\x1b[0m",`Succeeded processing link ${link.url} for user ${link.collection.ownerId}.`);- } catch (error) {+ } catch (error: any) {console.error("\x1b[34m%s\x1b[0m",`Error processing link ${link.url} for user ${link.collection.ownerId}:`,error);++ if (!browser.isConnected?.()) {+ await restartBrowser("browser disconnected");+ }}};- // Process each link in the batch concurrentlyconst processingPromises = links.map((e) => archiveLink(e));-await Promise.allSettled(processingPromises);const unprocessedLinkCount = await prisma.link.count({- where: {- lastPreserved: null,- url: { not: null },- },+ where: { lastPreserved: null, url: { not: null } },});console.log(