A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
refactor(workers): move media stripping to inside the parser (#3163)
Mohamed Bassem43c1ae26c33e
diff --git a/apps/workers/scripts/parseHtmlSubprocess.ts b/apps/workers/scripts/parseHtmlSubprocess.tsindex 273ec890..b90a9f34 100644--- a/apps/workers/scripts/parseHtmlSubprocess.ts+++ b/apps/workers/scripts/parseHtmlSubprocess.ts@@ -1,3 +1,5 @@+import { readFile } from "node:fs/promises";+import { Readability } from "@mozilla/readability";import DOMPurify from "dompurify";import { HttpProxyAgent } from "http-proxy-agent";@@ -175,6 +177,40 @@ function extractReadableContent(}}+const EMBEDDED_MEDIA_DATA_URI_PATTERN = /data:(?:audio|video)\/[^"'\s<>)]*/gi;++/**+ * Full-page archives can contain large audio and video files embedded directly+ * in src attributes. Those payloads are useful in the stored archive, but they+ * are irrelevant to metadata and readable-content extraction and cause large+ * memory amplification when parsed into a DOM.+ *+ * Replace them only in the parser's in-memory copy. The original HTML file+ * remains untouched for full-page archival.+ */+function replaceEmbeddedMediaDataUris(htmlContent: string): {+ htmlContent: string;+ replacedBytes: number;+ replacementCount: number;+} {+ let replacedBytes = 0;+ let replacementCount = 0;+ const parserHtmlContent = htmlContent.replace(+ EMBEDDED_MEDIA_DATA_URI_PATTERN,+ (dataUri) => {+ replacedBytes += dataUri.length;+ replacementCount += 1;+ return "about:blank";+ },+ );++ return {+ htmlContent: parserHtmlContent,+ replacedBytes,+ replacementCount,+ };+}+async function main() {// Read all of stdinconst chunks: Buffer[] = [];@@ -184,7 +220,16 @@ async function main() {const input = parseSubprocessInputSchema.parse(JSON.parse(Buffer.concat(chunks).toString()),);- const { htmlContent, url, jobId, metadataOnly } = input;+ const { htmlPath, url, jobId, metadataOnly } = input;+ const parserInput = replaceEmbeddedMediaDataUris(+ await readFile(htmlPath, "utf8"),+ );+ if (parserInput.replacementCount > 0) {+ logger.info(+ `[Crawler][${jobId}] Replaced ${parserInput.replacementCount} embedded audio/video data URIs (${parserInput.replacedBytes} bytes) before parsing.`,+ );+ }+ const { htmlContent } = parserInput;logger.info(`[Crawler][${jobId}] Will attempt to extract metadata from page ...`,diff --git a/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts b/apps/workers/workers/utils/parseHtmlSubprocessIpc.tsindex dbac019c..b8fea427 100644--- a/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts+++ b/apps/workers/workers/utils/parseHtmlSubprocessIpc.ts@@ -6,7 +6,9 @@ import {} from "@karakeep/shared/types/bookmarks";export const parseSubprocessInputSchema = z.object({- htmlContent: z.string(),+ // Path to a temp file containing the HTML to parse. The parent process+ // owns this file and deletes it once the subprocess exits.+ htmlPath: z.string(),url: z.string(),jobId: z.string(),// When true, only run metadata extraction and skip the (expensive)diff --git a/apps/workers/workers/utils/parseSubprocess.ts b/apps/workers/workers/utils/parseSubprocess.tsindex 84e6ded2..8b7ce190 100644--- a/apps/workers/workers/utils/parseSubprocess.ts+++ b/apps/workers/workers/utils/parseSubprocess.ts@@ -1,12 +1,16 @@// Runs the HTML parsing (metadata extraction + readability) in a separate// node process with a bounded heap, so a pathological page can't OOM the// worker itself.+import * as fs from "node:fs/promises";+import * as os from "node:os";+import * as path from "node:path";import { execa } from "execa";import { getTracer, withSpan } from "@karakeep/shared-server";import { getBookmarkDomain } from "network";import serverConfig from "@karakeep/shared/config";import logger from "@karakeep/shared/logger";+import { tryCatch } from "@karakeep/shared/tryCatch";import type {ParseSubprocessError,@@ -20,40 +24,6 @@ import { truncateUrl } from "./crawlerUtils";const tracer = getTracer("@karakeep/workers");-const EMBEDDED_MEDIA_DATA_URI_PATTERN = /data:(?:audio|video)\/[^"'\s<>)]*/gi;--/**- * Full-page archives can contain large audio and video files embedded directly- * in src attributes. Those payloads are useful in the stored archive, but they- * are irrelevant to metadata and readable-content extraction and cause large- * memory amplification when parsed into a DOM.- *- * Replace them only in the copy sent to the parser subprocess. The original- * HTML remains untouched for full-page archival.- */-export function replaceEmbeddedMediaDataUris(htmlContent: string): {- htmlContent: string;- replacedBytes: number;- replacementCount: number;-} {- let replacedBytes = 0;- let replacementCount = 0;- const parserHtmlContent = htmlContent.replace(- EMBEDDED_MEDIA_DATA_URI_PATTERN,- (dataUri) => {- replacedBytes += dataUri.length;- replacementCount += 1;- return "about:blank";- },- );-- return {- htmlContent: parserHtmlContent,- replacedBytes,- replacementCount,- };-}-function getSubprocessScriptPath(): string {const currentUrl = import.meta.url;if (currentUrl.includes("/dist/")) {@@ -110,25 +80,33 @@ export async function runParseSubprocess(const { cmd, args } = getSubprocessCommand();const timeoutMs = serverConfig.crawler.parseTimeoutSec * 1000;- const parserInput = replaceEmbeddedMediaDataUris(htmlContent);- if (parserInput.replacementCount > 0) {- logger.info(- `[Crawler][${jobId}] Replaced ${parserInput.replacementCount} embedded audio/video data URIs (${parserInput.replacedBytes} bytes) before parsing.`,- );- }- const result = await execa({- input: JSON.stringify({- htmlContent: parserInput.htmlContent,- url,- jobId,- metadataOnly: opts?.metadataOnly,- }),- cancelSignal: abortSignal,- timeout: timeoutMs,- reject: false,- stderr: "inherit",- })(cmd, args);+ // Hand the HTML to the subprocess via a temp file instead of stdin. The+ // file is only needed while the subprocess runs, so it's removed as soon+ // as it exits (including on abort/timeout).+ const tmpDir = await fs.mkdtemp(+ path.join(os.tmpdir(), "karakeep-parse-"),+ );+ const htmlPath = path.join(tmpDir, "page.html");+ const result = await (async () => {+ try {+ await fs.writeFile(htmlPath, htmlContent, "utf8");+ return await execa({+ input: JSON.stringify({+ htmlPath,+ url,+ jobId,+ metadataOnly: opts?.metadataOnly,+ }),+ cancelSignal: abortSignal,+ timeout: timeoutMs,+ reject: false,+ stderr: "inherit",+ })(cmd, args);+ } finally {+ await tryCatch(fs.rm(tmpDir, { recursive: true, force: true }));+ }+ })();if (result.isCanceled) {throw new Error(