Skip to content

⚡️⚡️⚡️ Self-hosted collaborative bookmark manager to collect, read, annotate, and fully preserve what matters, all in one place.

fix(worker): exit monolith processes appropriately

daniel31x13d647c60f04b8
diff --git a/apps/worker/lib/archiveHandler.ts b/apps/worker/lib/archiveHandler.ts
index 250200df..63e067f7 100644
--- a/apps/worker/lib/archiveHandler.ts
+++ b/apps/worker/lib/archiveHandler.ts
@@ -57,16 +57,18 @@ export default async function archiveHandler(
return;
}
+ const abortController = new AbortController();
+
const timeoutPromise = new Promise((_, reject) => {
- setTimeout(
- () =>
- reject(
- new Error(
- `Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`
- )
- ),
- BROWSER_TIMEOUT * 60000
- );
+ setTimeout(() => {
+ abortController.abort();
+
+ return reject(
+ new Error(
+ `Browser has been open for more than ${BROWSER_TIMEOUT} minutes.`
+ )
+ );
+ }, BROWSER_TIMEOUT * 60000);
});
const { browser, context } = await getBrowser();
@@ -139,6 +141,15 @@ export default async function archiveHandler(
if (archivalSettings.archiveAsReadable && !link.readable)
await handleReadability(content, link);
+ // Screenshot/PDF
+ if (
+ (archivalSettings.archiveAsScreenshot && !link.image) ||
+ (archivalSettings.archiveAsPDF && !link.pdf)
+ )
+ await handleScreenshotAndPdf(link, page, archivalSettings);
+
+ await browser.close();
+
// Auto-tagging
if (
archivalSettings.aiTag &&
@@ -152,23 +163,16 @@ export default async function archiveHandler(
)
await autoTagLink(user, link.id, metaDescription);
- // Screenshot/PDF
- if (
- (archivalSettings.archiveAsScreenshot && !link.image) ||
- (archivalSettings.archiveAsPDF && !link.pdf)
- )
- await handleScreenshotAndPdf(link, page, archivalSettings);
-
// Monolith
if (archivalSettings.archiveAsMonolith && !link.monolith && link.url)
- await handleMonolith(link, content);
+ await handleMonolith(link, content, abortController.signal);
}
})(),
timeoutPromise,
]);
} catch (err) {
- console.log(err);
- console.log("Failed Link details:", link);
+ console.log("Failed Link:", link.url);
+ console.log("Reason:", err);
throw err;
} finally {
const finalLink = await prisma.link.findUnique({
@@ -198,7 +202,9 @@ export default async function archiveHandler(
await removeFiles(link.id, link.collectionId);
}
- await browser.close();
+ if (browser && browser.isConnected()) {
+ await browser.close();
+ }
}
}
diff --git a/apps/worker/lib/autoTagLink.ts b/apps/worker/lib/autoTagLink.ts
index 1ee98d3e..1bebe2e1 100644
--- a/apps/worker/lib/autoTagLink.ts
+++ b/apps/worker/lib/autoTagLink.ts
@@ -81,13 +81,6 @@ export default async function autoTagLink(
if (!description) return;
- console.log(
- 'Auto tagging "',
- link.url,
- '" with the following description: ',
- description
- );
-
let prompt;
let existingTagsNames: string[] = [];
diff --git a/apps/worker/lib/preservationScheme/handleMonolith.ts b/apps/worker/lib/preservationScheme/handleMonolith.ts
index 4d7d289a..52cb362c 100644
--- a/apps/worker/lib/preservationScheme/handleMonolith.ts
+++ b/apps/worker/lib/preservationScheme/handleMonolith.ts
@@ -1,47 +1,92 @@
-import { execSync } from "child_process";
+import { spawn } from "child_process";
import { createFile } from "@linkwarden/filesystem";
import { prisma } from "@linkwarden/prisma";
import { Link } from "@prisma/client";
-const handleMonolith = async (link: Link, content: string) => {
+export default async function handleMonolith(
+ link: Link,
+ htmlFromPage: string,
+ signal: AbortSignal
+): Promise<void> {
if (!link.url) return;
- try {
- let html = execSync(
- `monolith - -I -b "${link.url}" ${
- process.env.MONOLITH_CUSTOM_OPTIONS || "-j -F -q"
- } -o -`,
- {
- timeout: 120000,
- killSignal: "SIGKILL",
- maxBuffer: 1024 * 1024 * Number(process.env.MONOLITH_MAX_BUFFER || 100),
- input: content,
+ return new Promise<void>((resolve, reject) => {
+ const args = [
+ "-",
+ "-I",
+ "-b",
+ link.url,
+ ...(process.env.MONOLITH_CUSTOM_OPTIONS
+ ? process.env.MONOLITH_CUSTOM_OPTIONS.split(" ")
+ : ["-j", "-F", "-q"]),
+ "-o",
+ "-",
+ ] as string[];
+
+ const child = spawn("monolith", args, {
+ stdio: ["pipe", "pipe", "inherit"],
+ detached: true,
+ });
+
+ const abortListener = () => {
+ try {
+ if (child.pid) killProcess(child.pid);
+ } catch {}
+ reject(new Error("Monolith aborted"));
+ };
+ signal?.addEventListener("abort", abortListener);
+
+ child.stdin.write(htmlFromPage);
+ child.stdin.end();
+
+ const chunks: Buffer[] = [];
+ child.stdout.on("data", (c) => chunks.push(c));
+
+ child.on("error", (err) => {
+ cleanup();
+ reject(err);
+ });
+
+ child.on("close", async (code) => {
+ cleanup();
+
+ if (code !== 0) {
+ return reject(new Error(`Monolith exited with code ${code}`));
}
- );
- if (!html?.length)
- return console.error("Error archiving as Monolith: Empty buffer");
+ const html = Buffer.concat(chunks);
+ if (!html.length) {
+ return reject(new Error("Monolith produced an empty file"));
+ }
+
+ const max = 1024 * 1024 * Number(process.env.MONOLITH_MAX_BUFFER || 6);
+ if (html.length > max) {
+ return reject(new Error("Monolith output exceeded buffer limit"));
+ }
- if (
- Buffer.byteLength(html) >
- 1024 * 1024 * Number(process.env.MONOLITH_MAX_BUFFER || 6)
- )
- return console.error("Error archiving as Monolith: Buffer size exceeded");
+ await createFile({
+ data: html,
+ filePath: `archives/${link.collectionId}/${link.id}.html`,
+ });
- await createFile({
- data: html,
- filePath: `archives/${link.collectionId}/${link.id}.html`,
- }).then(async () => {
await prisma.link.update({
where: { id: link.id },
- data: {
- monolith: `archives/${link.collectionId}/${link.id}.html`,
- },
+ data: { monolith: `archives/${link.collectionId}/${link.id}.html` },
});
+
+ resolve();
});
- } catch (err) {
- console.log("Uncaught Monolith error...");
+
+ function cleanup() {
+ signal?.removeEventListener("abort", abortListener);
+ }
+ });
+}
+
+const killProcess = async (PID: number) => {
+ if (process.platform === "win32") {
+ process.kill(PID, "SIGKILL");
+ } else {
+ process.kill(-PID, "SIGKILL");
}
};
-
-export default handleMonolith;
diff --git a/apps/worker/lib/preservationScheme/sendToWayback.ts b/apps/worker/lib/preservationScheme/sendToWayback.ts
index d264c785..a02d2cb7 100644
--- a/apps/worker/lib/preservationScheme/sendToWayback.ts
+++ b/apps/worker/lib/preservationScheme/sendToWayback.ts
@@ -16,8 +16,5 @@ export default async function sendToWayback(url: string) {
})
.then((response) => {
console.log(`Sent ${url} to Wayback Machine`);
- })
- .catch((error) => {
- console.error(error);
});
}