A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
fix: parallelize content extraction in crawler worker
Mohamed Bassem74c34529ade3
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.tsindex 97d1abb9..8eaf5d50 100644--- a/apps/workers/workers/crawlerWorker.ts+++ b/apps/workers/workers/crawlerWorker.ts@@ -463,7 +463,6 @@ async function crawlPage(},},async () => {- // Check user's browser crawling settingconst userData = await db.query.users.findFirst({where: eq(users.id, userId),columns: { browserCrawlingEnabled: true },@@ -480,11 +479,21 @@ async function crawlPage(}let browser: Browser | undefined;- if (serverConfig.crawler.browserConnectOnDemand) {- browser = await startBrowserInstance();- } else {- browser = globalBrowser;- }+ browser = await withSpan(+ tracer,+ "crawlerWorker.crawlPage.getBrowserInstance",+ {+ attributes: {+ "job.id": jobId,+ },+ },+ async () => {+ if (serverConfig.crawler.browserConnectOnDemand) {+ return startBrowserInstance();+ }+ return globalBrowser;+ },+ );if (!browser) {return browserlessCrawlPage(jobId, url, abortSignal);}@@ -493,12 +502,22 @@ async function crawlPage(const isRunningInProxyContext =proxyConfig !== undefined &&!matchesNoProxy(url, proxyConfig.bypass?.split(",") ?? []);- const context = await browser.newContext({- viewport: { width: 1440, height: 900 },- userAgent:- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",- proxy: proxyConfig,- });+ const context = await withSpan(+ tracer,+ "crawlerWorker.crawlPage.createContext",+ {+ attributes: {+ "job.id": jobId,+ },+ },+ async () =>+ browser.newContext({+ viewport: { width: 1440, height: 900 },+ userAgent:+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",+ proxy: proxyConfig,+ }),+ );try {if (globalCookies.length > 0) {@@ -508,62 +527,85 @@ async function crawlPage();}- // Create a new page in the context- const page = await context.newPage();-- // Apply ad blocking- if (globalBlocker) {- await globalBlocker.enableBlockingInPage(page);- }-- // Block audio/video resources and disallowed sub-requests- await page.route("**/*", async (route) => {- if (abortSignal.aborted) {- await route.abort("aborted");- return;- }- const request = route.request();- const resourceType = request.resourceType();-- // Block audio/video resources- if (- resourceType === "media" ||- request.headers()["content-type"]?.includes("video/") ||- request.headers()["content-type"]?.includes("audio/")- ) {- await route.abort("aborted");- return;- }+ const page = await withSpan(+ tracer,+ "crawlerWorker.crawlPage.setupPage",+ {+ attributes: {+ "job.id": jobId,+ },+ },+ async () => {+ // Create a new page in the context+ const nextPage = await context.newPage();- const requestUrl = request.url();- const requestIsRunningInProxyContext =- proxyConfig !== undefined &&- !matchesNoProxy(requestUrl, proxyConfig.bypass?.split(",") ?? []);- if (- requestUrl.startsWith("http://") ||- requestUrl.startsWith("https://")- ) {- const validation = await validateUrl(- requestUrl,- requestIsRunningInProxyContext,- );- if (!validation.ok) {- logger.warn(- `[Crawler][${jobId}] Blocking sub-request to disallowed URL "${requestUrl}": ${validation.reason}`,- );- await route.abort("blockedbyclient");- return;+ // Apply ad blocking+ if (globalBlocker) {+ await globalBlocker.enableBlockingInPage(nextPage);}- }- // Continue with other requests- await route.continue();- });+ // Block audio/video resources and disallowed sub-requests+ await nextPage.route("**/*", async (route) => {+ if (abortSignal.aborted) {+ await route.abort("aborted");+ return;+ }+ const request = route.request();+ const resourceType = request.resourceType();++ // Block audio/video resources+ if (+ resourceType === "media" ||+ request.headers()["content-type"]?.includes("video/") ||+ request.headers()["content-type"]?.includes("audio/")+ ) {+ await route.abort("aborted");+ return;+ }++ const requestUrl = request.url();+ const requestIsRunningInProxyContext =+ proxyConfig !== undefined &&+ !matchesNoProxy(+ requestUrl,+ proxyConfig.bypass?.split(",") ?? [],+ );+ if (+ requestUrl.startsWith("http://") ||+ requestUrl.startsWith("https://")+ ) {+ const validation = await validateUrl(+ requestUrl,+ requestIsRunningInProxyContext,+ );+ if (!validation.ok) {+ logger.warn(+ `[Crawler][${jobId}] Blocking sub-request to disallowed URL "${requestUrl}": ${validation.reason}`,+ );+ await route.abort("blockedbyclient");+ return;+ }+ }++ // Continue with other requests+ await route.continue();+ });+ return nextPage;+ },+ );// Navigate to the target URL- const navigationValidation = await validateUrl(- url,- isRunningInProxyContext,+ const navigationValidation = await withSpan(+ tracer,+ "crawlerWorker.crawlPage.validateNavigationTarget",+ {+ attributes: {+ "job.id": jobId,+ "bookmark.url": url,+ "bookmark.domain": getBookmarkDomain(url),+ },+ },+ async () => validateUrl(url, isRunningInProxyContext),);if (!navigationValidation.ok) {throw new Error(@@ -572,112 +614,196 @@ async function crawlPage(}const targetUrl = navigationValidation.url.toString();logger.info(`[Crawler][${jobId}] Navigating to "${targetUrl}"`);- const response = await Promise.race([- page.goto(targetUrl, {- timeout: serverConfig.crawler.navigateTimeoutSec * 1000,- waitUntil: "domcontentloaded",- }),- abortPromise(abortSignal).then(() => null),- ]);+ const response = await withSpan(+ tracer,+ "crawlerWorker.crawlPage.navigate",+ {+ attributes: {+ "job.id": jobId,+ "bookmark.url": targetUrl,+ "bookmark.domain": getBookmarkDomain(targetUrl),+ },+ },+ async () =>+ Promise.race([+ page.goto(targetUrl, {+ timeout: serverConfig.crawler.navigateTimeoutSec * 1000,+ waitUntil: "domcontentloaded",+ }),+ abortPromise(abortSignal).then(() => null),+ ]),+ );+ setSpanAttributes({+ "crawler.statusCode": response?.status() ?? 0,+ });logger.info(`[Crawler][${jobId}] Successfully navigated to "${targetUrl}". Waiting for the page to load ...`,);// Wait until network is relatively idle or timeout after 5 seconds- await Promise.race([- page- .waitForLoadState("networkidle", { timeout: 5000 })- .catch(() => ({})),- new Promise((resolve) => setTimeout(resolve, 5000)),- abortPromise(abortSignal),- ]);-- abortSignal.throwIfAborted();-- logger.info(- `[Crawler][${jobId}] Finished waiting for the page to load.`,+ await withSpan(+ tracer,+ "crawlerWorker.crawlPage.waitForLoadState",+ {+ attributes: {+ "job.id": jobId,+ "bookmark.url": targetUrl,+ "bookmark.domain": getBookmarkDomain(targetUrl),+ },+ },+ async () => {+ await Promise.race([+ page+ .waitForLoadState("networkidle", { timeout: 5000 })+ .catch(() => ({})),+ new Promise((resolve) => setTimeout(resolve, 5000)),+ abortPromise(abortSignal),+ ]);+ },);- // Extract content from the page- const htmlContent = await page.content();-abortSignal.throwIfAborted();logger.info(- `[Crawler][${jobId}] Successfully fetched the page content.`,+ `[Crawler][${jobId}] Finished waiting for the page to load.`,);- // Take a screenshot if configured- let screenshot: Buffer | undefined = undefined;- if (serverConfig.crawler.storeScreenshot) {- const { data: screenshotData, error: screenshotError } =- await tryCatch(- Promise.race<Buffer>([- page.screenshot({- // If you change this, you need to change the asset type in the store function.- type: "jpeg",- fullPage: serverConfig.crawler.fullPageScreenshot,- quality: 80,- }),- new Promise((_, reject) =>- setTimeout(- () =>- reject(- "TIMED_OUT, consider increasing CRAWLER_SCREENSHOT_TIMEOUT_SEC",- ),- serverConfig.crawler.screenshotTimeoutSec * 1000,- ),- ),- abortPromise(abortSignal).then(() => Buffer.from("")),- ]),- );- abortSignal.throwIfAborted();- if (screenshotError) {- logger.warn(- `[Crawler][${jobId}] Failed to capture the screenshot. Reason: ${screenshotError}`,- );- } else {- logger.info(- `[Crawler][${jobId}] Finished capturing page content and a screenshot. FullPageScreenshot: ${serverConfig.crawler.fullPageScreenshot}`,+ const [htmlContent, screenshot, pdf] = await withSpan(+ tracer,+ "crawlerWorker.crawlPage.captureAssets",+ {+ attributes: {+ "job.id": jobId,+ },+ },+ async () => {+ const htmlPromise = withSpan(+ tracer,+ "crawlerWorker.crawlPage.extractHtml",+ {+ attributes: {+ "job.id": jobId,+ },+ },+ async () => {+ const content = await page.content();+ abortSignal.throwIfAborted();+ logger.info(+ `[Crawler][${jobId}] Successfully fetched the page content.`,+ );+ return content;+ },);- screenshot = screenshotData;- }- }- // Capture PDF if configured or explicitly requested- let pdf: Buffer | undefined = undefined;- if (serverConfig.crawler.storePdf || forceStorePdf) {- const { data: pdfData, error: pdfError } = await tryCatch(- Promise.race<Buffer>([- page.pdf({- format: "A4",- printBackground: true,- }),- new Promise((_, reject) =>- setTimeout(- () =>- reject(- "TIMED_OUT, consider increasing CRAWLER_SCREENSHOT_TIMEOUT_SEC",- ),- serverConfig.crawler.screenshotTimeoutSec * 1000,- ),- ),- abortPromise(abortSignal).then(() => Buffer.from("")),- ]),- );- abortSignal.throwIfAborted();- if (pdfError) {- logger.warn(- `[Crawler][${jobId}] Failed to capture the PDF. Reason: ${pdfError}`,- );- } else {- logger.info(- `[Crawler][${jobId}] Finished capturing page content as PDF`,- );- pdf = pdfData;- }- }+ const screenshotPromise: Promise<Buffer | undefined> = serverConfig+ .crawler.storeScreenshot+ ? withSpan(+ tracer,+ "crawlerWorker.crawlPage.captureScreenshot",+ {+ attributes: {+ "job.id": jobId,+ "asset.type": "image",+ },+ },+ async () => {+ const { data: screenshotData, error: screenshotError } =+ await tryCatch(+ Promise.race<Buffer>([+ page.screenshot({+ // If you change this, you need to change the asset type in the store function.+ type: "jpeg",+ fullPage: serverConfig.crawler.fullPageScreenshot,+ quality: 80,+ }),+ new Promise((_, reject) =>+ setTimeout(+ () =>+ reject(+ "TIMED_OUT, consider increasing CRAWLER_SCREENSHOT_TIMEOUT_SEC",+ ),+ serverConfig.crawler.screenshotTimeoutSec * 1000,+ ),+ ),+ abortPromise(abortSignal).then(() => Buffer.from("")),+ ]),+ );+ abortSignal.throwIfAborted();+ if (screenshotError) {+ logger.warn(+ `[Crawler][${jobId}] Failed to capture the screenshot. Reason: ${screenshotError}`,+ );+ return undefined;+ }+ setSpanAttributes({+ "asset.size": screenshotData.byteLength,+ });+ logger.info(+ `[Crawler][${jobId}] Finished capturing page content and a screenshot. FullPageScreenshot: ${serverConfig.crawler.fullPageScreenshot}`,+ );+ return screenshotData;+ },+ )+ : Promise.resolve(undefined);++ const pdfPromise: Promise<Buffer | undefined> =+ serverConfig.crawler.storePdf || forceStorePdf+ ? withSpan(+ tracer,+ "crawlerWorker.crawlPage.capturePdf",+ {+ attributes: {+ "job.id": jobId,+ "asset.type": "pdf",+ },+ },+ async () => {+ const { data: pdfData, error: pdfError } = await tryCatch(+ Promise.race<Buffer>([+ page.pdf({+ format: "A4",+ printBackground: true,+ }),+ new Promise((_, reject) =>+ setTimeout(+ () =>+ reject(+ "TIMED_OUT, consider increasing CRAWLER_SCREENSHOT_TIMEOUT_SEC",+ ),+ serverConfig.crawler.screenshotTimeoutSec * 1000,+ ),+ ),+ abortPromise(abortSignal).then(() => Buffer.from("")),+ ]),+ );+ abortSignal.throwIfAborted();+ if (pdfError) {+ logger.warn(+ `[Crawler][${jobId}] Failed to capture the PDF. Reason: ${pdfError}`,+ );+ return undefined;+ }+ setSpanAttributes({+ "asset.size": pdfData.byteLength,+ });+ logger.info(+ `[Crawler][${jobId}] Finished capturing page content as PDF`,+ );+ return pdfData;+ },+ )+ : Promise.resolve(undefined);++ const captureResults = await Promise.all([+ htmlPromise,+ screenshotPromise,+ pdfPromise,+ ] as const);+ abortSignal.throwIfAborted();+ return captureResults;+ },+ );return {htmlContent,