⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.
TypeScript bookmarkbookmark-managercollaborationnextjsreact-nativeread-it-laterself-hostedtypescript
fix(worker): exit monolith processes appropriately
daniel31x13d647c60f04b8
diff --git a/apps/worker/lib/archiveHandler.ts b/apps/worker/lib/archiveHandler.tsindex 250200df..63e067f7 100644--- a/apps/worker/lib/archiveHandler.ts+++ b/apps/worker/lib/archiveHandler.ts@@ -57,16 +57,18 @@ export default async function archiveHandler(return;}+ const abortController = new AbortController();+const timeoutPromise = new Promise((_, reject) => {- setTimeout(- () =>- reject(- new Error(- `Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`- )- ),- BROWSER_TIMEOUT * 60000- );+ setTimeout(() => {+ abortController.abort();++ return reject(+ new Error(+ `Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`+ )+ );+ }, BROWSER_TIMEOUT * 60000);});const { browser, context } = await getBrowser();@@ -139,6 +141,15 @@ export default async function archiveHandler(if (archivalSettings.archiveAsReadable && !link.readable)await handleReadability(content, link);+ // Screenshot/PDF+ if (+ (archivalSettings.archiveAsScreenshot && !link.image) ||+ (archivalSettings.archiveAsPDF && !link.pdf)+ )+ await handleScreenshotAndPdf(link, page, archivalSettings);++ await browser.close();+// Auto-taggingif (archivalSettings.aiTag &&@@ -152,23 +163,16 @@ export default async function archiveHandler()await autoTagLink(user, link.id, metaDescription);- // Screenshot/PDF- if (- (archivalSettings.archiveAsScreenshot && !link.image) ||- (archivalSettings.archiveAsPDF && !link.pdf)- )- await handleScreenshotAndPdf(link, page, archivalSettings);-// Monolithif (archivalSettings.archiveAsMonolith && !link.monolith && link.url)- await handleMonolith(link, content);+ await handleMonolith(link, content, abortController.signal);}})(),timeoutPromise,]);} catch (err) {- console.log(err);- console.log("Failed Link details:", link);+ console.log("Failed Link:", link.url);+ console.log("Reason:", err);throw err;} finally {const finalLink = await prisma.link.findUnique({@@ -198,7 +202,9 @@ export default async function archiveHandler(await removeFiles(link.id, link.collectionId);}- await browser.close();+ if (browser && browser.isConnected()) {+ await browser.close();+ }}}diff --git a/apps/worker/lib/autoTagLink.ts b/apps/worker/lib/autoTagLink.tsindex 1ee98d3e..1bebe2e1 100644--- a/apps/worker/lib/autoTagLink.ts+++ b/apps/worker/lib/autoTagLink.ts@@ -81,13 +81,6 @@ export default async function autoTagLink(if (!description) return;- console.log(- 'Auto tagging "',- link.url,- '" with the following description: ',- description- );-let prompt;let existingTagsNames: string[] = [];diff --git a/apps/worker/lib/preservationScheme/handleMonolith.ts b/apps/worker/lib/preservationScheme/handleMonolith.tsindex 4d7d289a..52cb362c 100644--- a/apps/worker/lib/preservationScheme/handleMonolith.ts+++ b/apps/worker/lib/preservationScheme/handleMonolith.ts@@ -1,47 +1,92 @@-import { execSync } from "child_process";+import { spawn } from "child_process";import { createFile } from "@linkwarden/filesystem";import { prisma } from "@linkwarden/prisma";import { Link } from "@prisma/client";-const handleMonolith = async (link: Link, content: string) => {+export default async function handleMonolith(+ link: Link,+ htmlFromPage: string,+ signal: AbortSignal+): Promise<void> {if (!link.url) return;- try {- let html = execSync(- `monolith - -I -b "${link.url}" ${- process.env.MONOLITH_CUSTOM_OPTIONS || "-j -F -q"- } -o -`,- {- timeout: 120000,- killSignal: "SIGKILL",- maxBuffer: 1024 * 1024 * Number(process.env.MONOLITH_MAX_BUFFER || 100),- input: content,+ return new Promise<void>((resolve, reject) => {+ const args = [+ "-",+ "-I",+ "-b",+ link.url,+ ...(process.env.MONOLITH_CUSTOM_OPTIONS+ ? process.env.MONOLITH_CUSTOM_OPTIONS.split(" ")+ : ["-j", "-F", "-q"]),+ "-o",+ "-",+ ] as string[];++ const child = spawn("monolith", args, {+ stdio: ["pipe", "pipe", "inherit"],+ detached: true,+ });++ const abortListener = () => {+ try {+ if (child.pid) killProcess(child.pid);+ } catch {}+ reject(new Error("Monolith aborted"));+ };+ signal?.addEventListener("abort", abortListener);++ child.stdin.write(htmlFromPage);+ child.stdin.end();++ const chunks: Buffer[] = [];+ child.stdout.on("data", (c) => chunks.push(c));++ child.on("error", (err) => {+ cleanup();+ reject(err);+ });++ child.on("close", async (code) => {+ cleanup();++ if (code !== 0) {+ return reject(new Error(`Monolith exited with code ${code}`));}- );- if (!html?.length)- return console.error("Error archiving as Monolith: Empty buffer");+ const html = Buffer.concat(chunks);+ if (!html.length) {+ return reject(new Error("Monolith produced an empty file"));+ }++ const max = 1024 * 1024 * Number(process.env.MONOLITH_MAX_BUFFER || 6);+ if (html.length > max) {+ return reject(new Error("Monolith output exceeded buffer limit"));+ }- if (- Buffer.byteLength(html) >- 1024 * 1024 * Number(process.env.MONOLITH_MAX_BUFFER || 6)- )- return console.error("Error archiving as Monolith: Buffer size exceeded");+ await createFile({+ data: html,+ filePath: `archives/${link.collectionId}/${link.id}.html`,+ });- await createFile({- data: html,- filePath: `archives/${link.collectionId}/${link.id}.html`,- }).then(async () => {await prisma.link.update({where: { id: link.id },- data: {- monolith: `archives/${link.collectionId}/${link.id}.html`,- },+ data: { monolith: `archives/${link.collectionId}/${link.id}.html` },});++ resolve();});- } catch (err) {- console.log("Uncaught Monolith error...");++ function cleanup() {+ signal?.removeEventListener("abort", abortListener);+ }+ });+}++const killProcess = async (PID: number) => {+ if (process.platform === "win32") {+ process.kill(PID, "SIGKILL");+ } else {+ process.kill(-PID, "SIGKILL");}};--export default handleMonolith;diff --git a/apps/worker/lib/preservationScheme/sendToWayback.ts b/apps/worker/lib/preservationScheme/sendToWayback.tsindex d264c785..a02d2cb7 100644--- a/apps/worker/lib/preservationScheme/sendToWayback.ts+++ b/apps/worker/lib/preservationScheme/sendToWayback.ts@@ -16,8 +16,5 @@ export default async function sendToWayback(url: string) {}).then((response) => {console.log(`Sent ${url} to Wayback Machine`);- })- .catch((error) => {- console.error(error);});}