Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

refactor(workers): move media stripping to inside the parser (#3163)

Mohamed Bassem43c1ae26c33e
diff --git a/apps/workers/scripts/parseHtmlSubprocess.ts b/apps/workers/scripts/parseHtmlSubprocess.ts
index 273ec890..b90a9f34 100644
--- a/apps/workers/scripts/parseHtmlSubprocess.ts
+++ b/apps/workers/scripts/parseHtmlSubprocess.ts
@@ -1,3 +1,5 @@
+import { readFile } from "node:fs/promises";
+
import { Readability } from "@mozilla/readability";
import DOMPurify from "dompurify";
import { HttpProxyAgent } from "http-proxy-agent";
@@ -175,6 +177,40 @@ function extractReadableContent(
}
}
+const EMBEDDED_MEDIA_DATA_URI_PATTERN = /data:(?:audio|video)\/[^"'\s<>)]*/gi;
+
+/**
+ * Full-page archives can contain large audio and video files embedded directly
+ * in src attributes. Those payloads are useful in the stored archive, but they
+ * are irrelevant to metadata and readable-content extraction and cause large
+ * memory amplification when parsed into a DOM.
+ *
+ * Replace them only in the parser's in-memory copy. The original HTML file
+ * remains untouched for full-page archival.
+ */
+function replaceEmbeddedMediaDataUris(htmlContent: string): {
+ htmlContent: string;
+ replacedBytes: number;
+ replacementCount: number;
+} {
+ let replacedBytes = 0;
+ let replacementCount = 0;
+ const parserHtmlContent = htmlContent.replace(
+ EMBEDDED_MEDIA_DATA_URI_PATTERN,
+ (dataUri) => {
+ replacedBytes += dataUri.length;
+ replacementCount += 1;
+ return "about:blank";
+ },
+ );
+
+ return {
+ htmlContent: parserHtmlContent,
+ replacedBytes,
+ replacementCount,
+ };
+}
+
async function main() {
// Read all of stdin
const chunks: Buffer[] = [];
@@ -184,7 +220,16 @@ async function main() {
const input = parseSubprocessInputSchema.parse(
JSON.parse(Buffer.concat(chunks).toString()),
);
- const { htmlContent, url, jobId, metadataOnly } = input;
+ const { htmlPath, url, jobId, metadataOnly } = input;
+ const parserInput = replaceEmbeddedMediaDataUris(
+ await readFile(htmlPath, "utf8"),
+ );
+ if (parserInput.replacementCount > 0) {
+ logger.info(
+ `[Crawler][${jobId}] Replaced ${parserInput.replacementCount} embedded audio/video data URIs (${parserInput.replacedBytes} bytes) before parsing.`,
+ );
+ }
+ const { htmlContent } = parserInput;
logger.info(
`[Crawler][${jobId}] Will attempt to extract metadata from page ...`,
diff --git a/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts b/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts
index dbac019c..b8fea427 100644
--- a/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts
+++ b/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts
@@ -6,7 +6,9 @@ import {
} from "@karakeep/shared/types/bookmarks";
export const parseSubprocessInputSchema = z.object({
- htmlContent: z.string(),
+ // Path to a temp file containing the HTML to parse. The parent process
+ // owns this file and deletes it once the subprocess exits.
+ htmlPath: z.string(),
url: z.string(),
jobId: z.string(),
// When true, only run metadata extraction and skip the (expensive)
diff --git a/apps/workers/workers/utils/parseSubprocess.ts b/apps/workers/workers/utils/parseSubprocess.ts
index 84e6ded2..8b7ce190 100644
--- a/apps/workers/workers/utils/parseSubprocess.ts
+++ b/apps/workers/workers/utils/parseSubprocess.ts
@@ -1,12 +1,16 @@
// Runs the HTML parsing (metadata extraction + readability) in a separate
// node process with a bounded heap, so a pathological page can't OOM the
// worker itself.
+import * as fs from "node:fs/promises";
+import * as os from "node:os";
+import * as path from "node:path";
import { execa } from "execa";
import { getTracer, withSpan } from "@karakeep/shared-server";
import { getBookmarkDomain } from "network";
import serverConfig from "@karakeep/shared/config";
import logger from "@karakeep/shared/logger";
+import { tryCatch } from "@karakeep/shared/tryCatch";
import type {
ParseSubprocessError,
@@ -20,40 +24,6 @@ import { truncateUrl } from "./crawlerUtils";
const tracer = getTracer("@karakeep/workers");
-const EMBEDDED_MEDIA_DATA_URI_PATTERN = /data:(?:audio|video)\/[^"'\s<>)]*/gi;
-
-/**
- * Full-page archives can contain large audio and video files embedded directly
- * in src attributes. Those payloads are useful in the stored archive, but they
- * are irrelevant to metadata and readable-content extraction and cause large
- * memory amplification when parsed into a DOM.
- *
- * Replace them only in the copy sent to the parser subprocess. The original
- * HTML remains untouched for full-page archival.
- */
-export function replaceEmbeddedMediaDataUris(htmlContent: string): {
- htmlContent: string;
- replacedBytes: number;
- replacementCount: number;
-} {
- let replacedBytes = 0;
- let replacementCount = 0;
- const parserHtmlContent = htmlContent.replace(
- EMBEDDED_MEDIA_DATA_URI_PATTERN,
- (dataUri) => {
- replacedBytes += dataUri.length;
- replacementCount += 1;
- return "about:blank";
- },
- );
-
- return {
- htmlContent: parserHtmlContent,
- replacedBytes,
- replacementCount,
- };
-}
-
function getSubprocessScriptPath(): string {
const currentUrl = import.meta.url;
if (currentUrl.includes("/dist/")) {
@@ -110,25 +80,33 @@ export async function runParseSubprocess(
const { cmd, args } = getSubprocessCommand();
const timeoutMs = serverConfig.crawler.parseTimeoutSec * 1000;
- const parserInput = replaceEmbeddedMediaDataUris(htmlContent);
- if (parserInput.replacementCount > 0) {
- logger.info(
- `[Crawler][${jobId}] Replaced ${parserInput.replacementCount} embedded audio/video data URIs (${parserInput.replacedBytes} bytes) before parsing.`,
- );
- }
- const result = await execa({
- input: JSON.stringify({
- htmlContent: parserInput.htmlContent,
- url,
- jobId,
- metadataOnly: opts?.metadataOnly,
- }),
- cancelSignal: abortSignal,
- timeout: timeoutMs,
- reject: false,
- stderr: "inherit",
- })(cmd, args);
+ // Hand the HTML to the subprocess via a temp file instead of stdin. The
+ // file is only needed while the subprocess runs, so it's removed as soon
+ // as it exits (including on abort/timeout).
+ const tmpDir = await fs.mkdtemp(
+ path.join(os.tmpdir(), "karakeep-parse-"),
+ );
+ const htmlPath = path.join(tmpDir, "page.html");
+ const result = await (async () => {
+ try {
+ await fs.writeFile(htmlPath, htmlContent, "utf8");
+ return await execa({
+ input: JSON.stringify({
+ htmlPath,
+ url,
+ jobId,
+ metadataOnly: opts?.metadataOnly,
+ }),
+ cancelSignal: abortSignal,
+ timeout: timeoutMs,
+ reject: false,
+ stderr: "inherit",
+ })(cmd, args);
+ } finally {
+ await tryCatch(fs.rm(tmpDir, { recursive: true, force: true }));
+ }
+ })();
if (result.isCanceled) {
throw new Error(