Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

feat(workers): screenshot precrawled archives (#3161)

Mohamed Bassembef14d780d76
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index a2366d64..273fe5d2 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -940,9 +940,10 @@ const crawlAndParseUrl = traced(
userId,
assetId: precrawledArchiveAssetId,
});
+ const htmlContent = asset.asset.toString();
result = {
- htmlContent: asset.asset.toString(),
- screenshot: undefined,
+ htmlContent,
+ screenshot: await screenshotPrecrawledArchive(ctx, htmlContent),
pdf: undefined,
statusCode: 200,
url,
@@ -1242,11 +1243,7 @@ export const crawlPage = traced(
return browserlessCrawlPage(ctx);
}
- const browser = await span("crawlPage.getBrowserInstance", async () =>
- serverConfig.crawler.browserConnectOnDemand
- ? startBrowserInstance()
- : getGlobalBrowser(),
- );
+ const browser = await getBrowserInstance();
if (!browser) {
return browserlessCrawlPage(ctx);
}
@@ -1372,6 +1369,117 @@ export const crawlPage = traced(
},
);
+const getBrowserInstance = traced(
+ "crawlPage.getBrowserInstance",
+ async (): Promise<Browser | undefined> =>
+ serverConfig.crawler.browserConnectOnDemand
+ ? startBrowserInstance()
+ : getGlobalBrowser(),
+);
+
+/**
+ * Renders a precrawled archive (e.g. a SingleFile capture uploaded by the
+ * browser extension) offline and screenshots it. The archive is untrusted
+ * user-provided HTML, so JavaScript is disabled and every network request is
+ * blocked; SingleFile archives inline their resources so they render fine
+ * without the network. Returns undefined when screenshots are disabled, the
+ * user can't use the browser, or no browser is available.
+ *
+ * The screenshot is best-effort: processing the archive itself doesn't need a
+ * browser, so browser failures are logged and skipped rather than failing the
+ * crawl. Aborts still propagate.
+ */
+const screenshotPrecrawledArchive = traced(
+ "screenshotPrecrawledArchive",
+ async (
+ ctx: CrawlContext,
+ htmlContent: string,
+ ): Promise<Buffer | undefined> => {
+ const { abortSignal, log } = ctx;
+ if (!serverConfig.crawler.storeScreenshot || !ctx.browserCrawlingEnabled) {
+ return undefined;
+ }
+ const { data: screenshot, error } = await tryCatch(
+ renderPrecrawledArchiveScreenshot(ctx, htmlContent),
+ );
+ abortSignal.throwIfAborted();
+ if (error) {
+ log.warn(
+ `Failed to screenshot the precrawled archive. Skipping the screenshot: ${error}`,
+ );
+ return undefined;
+ }
+ return screenshot;
+ },
+);
+
+async function renderPrecrawledArchiveScreenshot(
+ ctx: CrawlContext,
+ htmlContent: string,
+): Promise<Buffer | undefined> {
+ const { jobId, abortSignal, log } = ctx;
+ const browser = await getBrowserInstance();
+ if (!browser) {
+ log.info(
+ `No browser available. Skipping the screenshot of the precrawled archive.`,
+ );
+ return undefined;
+ }
+
+ let context: BrowserContext;
+ try {
+ context = await browser.newContext({
+ viewport: { width: 1440, height: 900 },
+ javaScriptEnabled: false,
+ serviceWorkers: "block",
+ offline: true,
+ });
+ } catch (e) {
+ // closePageAndContext would close an on-demand browser, but it's never
+ // reached without a context.
+ if (serverConfig.crawler.browserConnectOnDemand) {
+ await browser.close().catch((closeError: unknown) => {
+ log.warn(`browser.close() failed: ${closeError}`);
+ });
+ }
+ throw e;
+ }
+ trackContext(jobId, context);
+ let page: Page | undefined;
+ try {
+ page = await context.newPage();
+ await page.route("**/*", async (route) => {
+ const requestUrl = route.request().url();
+ if (requestUrl.startsWith("data:") || requestUrl === "about:blank") {
+ await route.fallback();
+ return;
+ }
+ await route.abort("blockedbyclient");
+ });
+
+ const { error: loadError } = await tryCatch(
+ raceWith(
+ page.setContent(htmlContent, {
+ timeout: serverConfig.crawler.navigateTimeoutSec * 1000,
+ waitUntil: "load",
+ }),
+ abortRace(abortSignal),
+ ),
+ );
+ abortSignal.throwIfAborted();
+ if (loadError) {
+ // A partially rendered page still makes a useful screenshot.
+ log.warn(
+ `Precrawled archive didn't fully load before screenshotting: ${loadError}`,
+ );
+ }
+
+ return await captureScreenshot(ctx, page);
+ } finally {
+ await closePageAndContext(ctx, page, context, browser);
+ }
+}
+
/**
* Creates and configures the page: redirect guard, adblocking, dialog
* auto-dismissal, media/SSRF request blocking, and abort wiring.
@@ -1475,6 +1583,46 @@ const setupPage = traced(
},
);
+/**
+ * Captures a JPEG screenshot of the page. Failures and timeouts are logged and
+ * reported as `undefined` since the screenshot is best-effort.
+ */
+const captureScreenshot = traced(
+ "crawlPage.captureScreenshot",
+ async (ctx: CrawlContext, activePage: Page): Promise<Buffer | undefined> => {
+ const { abortSignal, log } = ctx;
+ const { data: screenshotData, error: screenshotError } = await tryCatch(
+ raceWith<Buffer>(
+ activePage.screenshot({
+ // If you change this, change the content type in CAPTURED_ASSETS too.
+ type: "jpeg",
+ fullPage: serverConfig.crawler.fullPageScreenshot,
+ quality: 80,
+ }),
+ timeoutRace<Buffer>(
+ serverConfig.crawler.screenshotTimeoutSec * 1000,
+ () => {
+ throw new Error(
+ "TIMED_OUT, consider increasing CRAWLER_SCREENSHOT_TIMEOUT_SEC",
+ );
+ },
+ ),
+ abortRaceResolve(abortSignal, Buffer.from("")),
+ ),
+ );
+ abortSignal.throwIfAborted();
+ if (screenshotError) {
+ log.warn(`Failed to capture the screenshot. Reason: ${screenshotError}`);
+ return undefined;
+ }
+ setSpanAttributes({ "asset.size": screenshotData.byteLength });
+ log.info(
+ `Finished capturing page content and a screenshot. FullPageScreenshot: ${serverConfig.crawler.fullPageScreenshot}`,
+ );
+ return screenshotData;
+ },
+);
+
/**
* Extracts the page HTML and (depending on config) captures a screenshot and
* a PDF, all in parallel.
@@ -1494,42 +1642,8 @@ const capturePageAssets = traced(
return content;
});
- const screenshotPromise: Promise<Buffer | undefined> = serverConfig.crawler
- .storeScreenshot
- ? span("crawlPage.captureScreenshot", async () => {
- const { data: screenshotData, error: screenshotError } =
- await tryCatch(
- raceWith<Buffer>(
- activePage.screenshot({
- // If you change this, change the content type in CAPTURED_ASSETS too.
- type: "jpeg",
- fullPage: serverConfig.crawler.fullPageScreenshot,
- quality: 80,
- }),
- timeoutRace<Buffer>(
- serverConfig.crawler.screenshotTimeoutSec * 1000,
- () => {
- throw new Error(
- "TIMED_OUT, consider increasing CRAWLER_SCREENSHOT_TIMEOUT_SEC",
- );
- },
- ),
- abortRaceResolve(abortSignal, Buffer.from("")),
- ),
- );
- abortSignal.throwIfAborted();
- if (screenshotError) {
- log.warn(
- `Failed to capture the screenshot. Reason: ${screenshotError}`,
- );
- return undefined;
- }
- setSpanAttributes({ "asset.size": screenshotData.byteLength });
- log.info(
- `Finished capturing page content and a screenshot. FullPageScreenshot: ${serverConfig.crawler.fullPageScreenshot}`,
- );
- return screenshotData;
- })
+ const screenshotPromise = serverConfig.crawler.storeScreenshot
+ ? captureScreenshot(ctx, activePage)
: Promise.resolve(undefined);
const pdfPromise: Promise<Buffer | undefined> =