Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: provide URLs to the error snapshot (#2482)
Martin Adámek7f64145308df
diff --git a/packages/core/src/crawlers/error_snapshotter.ts b/packages/core/src/crawlers/error_snapshotter.tsindex dc773da13..48ef02092 100644--- a/packages/core/src/crawlers/error_snapshotter.ts+++ b/packages/core/src/crawlers/error_snapshotter.ts@@ -18,6 +18,13 @@ export interface SnapshotResult {htmlFileName?: string;}+interface ErrorSnapshot {+ screenshotFileName?: string;+ screenshotFileUrl?: string;+ htmlFileName?: string;+ htmlFileUrl?: string;+}+/*** ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.*@@ -42,7 +49,7 @@ export class ErrorSnapshotter {/*** Capture a snapshot of the error context.*/- async captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise<{ screenshotFileName?: string; htmlFileName?: string }> {+ async captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise<ErrorSnapshot> {try {const page = context?.page as BrowserPage | undefined;const body = context?.body;@@ -80,7 +87,9 @@ export class ErrorSnapshotter {return {screenshotFileName,+ screenshotFileUrl: screenshotFileName && keyValueStore.getPublicUrl(screenshotFileName),htmlFileName,+ htmlFileUrl: htmlFileName && keyValueStore.getPublicUrl(htmlFileName),};} catch {return {};diff --git a/packages/core/src/crawlers/error_tracker.ts b/packages/core/src/crawlers/error_tracker.tsindex e592ca846..5d22e8f6e 100644--- a/packages/core/src/crawlers/error_tracker.ts+++ b/packages/core/src/crawlers/error_tracker.ts@@ -407,10 +407,10 @@ export class ErrorTracker {return;}- const { screenshotFileName, htmlFileName } = await this.errorSnapshotter.captureSnapshot(error, context);+ const { screenshotFileUrl, htmlFileUrl } = await this.errorSnapshotter.captureSnapshot(error, context);- storage.firstErrorScreenshot = screenshotFileName;- storage.firstErrorHtml = htmlFileName;+ storage.firstErrorScreenshotUrl = screenshotFileUrl;+ storage.firstErrorHtmlUrl = htmlFileUrl;}reset() {diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.tsindex 6167e9192..12b157482 100644--- a/packages/core/src/storages/key_value_store.ts+++ b/packages/core/src/storages/key_value_store.ts@@ -428,6 +428,14 @@ export class KeyValueStore {: undefined; // [].forEach() returns undefined.}+ /**+ * Returns a file URL for the given key.+ */+ getPublicUrl(key: string): string {+ const name = this.name ?? this.config.get('defaultKeyValueStoreId');+ return `file://${process.cwd()}/storage/key_value_stores/${name}/${key}`;+ }+/*** Opens a key-value store and returns a promise resolving to an instance of the {@apilink KeyValueStore} class.*diff --git a/test/e2e/cheerio-error-snapshot/test.mjs b/test/e2e/cheerio-error-snapshot/test.mjsindex c36006291..912f6a7bf 100644--- a/test/e2e/cheerio-error-snapshot/test.mjs+++ b/test/e2e/cheerio-error-snapshot/test.mjs@@ -10,7 +10,7 @@ await expect(stats.requestsFailed === 4, 'All requests failed');let totalErrorHtmlFiles = 0;for (const error of Object.values(stats.errors)) {- if (hasNestedKey(error, 'firstErrorHtml')) {+ if (hasNestedKey(error, 'firstErrorHtmlUrl')) {totalErrorHtmlFiles++;}}diff --git a/test/e2e/puppeteer-error-snapshot/test.mjs b/test/e2e/puppeteer-error-snapshot/test.mjsindex 5a7a0121c..404d64e91 100644--- a/test/e2e/puppeteer-error-snapshot/test.mjs+++ b/test/e2e/puppeteer-error-snapshot/test.mjs@@ -11,13 +11,13 @@ await expect(stats.requestsFailed === 4, 'All requests failed');let totalErrorHtmlFiles = 0;let totalErrorScreenshotFiles = 0;for (const error of Object.values(stats.errors)) {- if (hasNestedKey(error, 'firstErrorHtml')) {+ if (hasNestedKey(error, 'firstErrorHtmlUrl')) {totalErrorHtmlFiles++;}}for (const error of Object.values(stats.errors)) {- if (hasNestedKey(error, 'firstErrorScreenshot')) {+ if (hasNestedKey(error, 'firstErrorScreenshotUrl')) {totalErrorScreenshotFiles++;}}