Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import crypto from 'node:crypto';
import type { KeyValueStore } from '@crawlee/core';
import type { CrawlingContext } from './crawler-commons.js';
import type { ErrnoException } from './error-tracker.js';
import type { SnapshottableProperties } from './internals/types.js';
// Define the following types as we cannot import the complete types from the respective packages
interface BrowserCrawlingContext {
saveSnapshot: (options: { key: string }) => Promise<void>;
}
interface SnapshotResult {
screenshotFileName?: string;
htmlFileName?: string;
}
interface ErrorSnapshot {
screenshotFileName?: string;
screenshotFileUrl?: string;
htmlFileName?: string;
htmlFileUrl?: string;
}
const MAX_ERROR_CHARACTERS = 30;
const MAX_HASH_LENGTH = 30;
const MAX_FILENAME_LENGTH = 250;
const BASE_MESSAGE = 'An error occurred';
const SNAPSHOT_PREFIX = 'ERROR_SNAPSHOT';
/**
* ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
*
* This functionality is opt-in, and can be enabled via the crawler options:
*
* ```ts
* const crawler = new BasicCrawler({
* // ...
* statistics: new Statistics({ saveErrorSnapshots: true }),
* });
* ```
*/
export class ErrorSnapshotter {
/**
* Capture a snapshot of the error context.
*/
async captureSnapshot(
error: ErrnoException,
context: CrawlingContext & SnapshottableProperties,
): Promise<ErrorSnapshot> {
try {
const page = context?.page;
const body = context?.body;
const keyValueStore = await context?.getKeyValueStore();
// If the key-value store is not available, or the body and page are not available, return empty filenames
if (!keyValueStore || (!body && !page)) {
return {};
}
const fileName = this.#generateFilename(error);
let screenshotFileName: string | undefined;
let htmlFileName: string | undefined;
if (page) {
const capturedFiles = await this.#contextCaptureSnapshot(
context as unknown as BrowserCrawlingContext,
fileName,
);
if (capturedFiles) {
screenshotFileName = capturedFiles.screenshotFileName;
htmlFileName = capturedFiles.htmlFileName;
}
// If the snapshot for browsers failed to capture the HTML, try to capture it from the page content
if (!htmlFileName) {
const html = await page.content();
htmlFileName = html ? await this.#saveHTMLSnapshot(html, keyValueStore, fileName) : undefined;
}
} else if (typeof body === 'string') {
// for non-browser contexts
htmlFileName = await this.#saveHTMLSnapshot(body, keyValueStore, fileName);
}
return {
screenshotFileName,
screenshotFileUrl: screenshotFileName && (await keyValueStore.getPublicUrl(screenshotFileName)),
htmlFileName,
htmlFileUrl: htmlFileName && (await keyValueStore.getPublicUrl(htmlFileName)),
};
} catch {
return {};
}
}
/**
* Captures a snapshot of the current page using the context.saveSnapshot function.
* This function is applicable for browser contexts only.
* Returns an object containing the filenames of the screenshot and HTML file.
*/
async #contextCaptureSnapshot(
context: BrowserCrawlingContext,
fileName: string,
): Promise<SnapshotResult | undefined> {
try {
await context.saveSnapshot({ key: fileName });
return {
screenshotFileName: `${fileName}.jpg`,
htmlFileName: `${fileName}.html`,
};
} catch {
return undefined;
}
}
/**
* Save the HTML snapshot of the page, and return the key it was stored under.
*/
async #saveHTMLSnapshot(
html: string,
keyValueStore: Pick<KeyValueStore, 'setValue'>,
fileName: string,
): Promise<string | undefined> {
try {
await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
// The record key is `fileName` - returning it with an `.html` suffix (as v3 did,
// where local storage put the extension in the key) would break `getPublicUrl`.
return fileName;
} catch {
return undefined;
}
}
/**
* Generate a unique fileName for each error snapshot.
*/
#generateFilename(error: ErrnoException): string {
// Create a hash of the error stack trace
const errorStackHash = crypto
.createHash('sha1')
.update(error.stack || error.message || '')
.digest('hex')
.slice(0, MAX_HASH_LENGTH);
const errorMessagePrefix = (error.message || BASE_MESSAGE).slice(0, MAX_ERROR_CHARACTERS).trim();
/**
* Remove non-word characters from the start and end of a string.
*/
const sanitizeString = (str: string): string => {
return str.replace(/^\W+|\W+$/g, '');
};
// Generate fileName and remove disallowed characters
const fileName = `${SNAPSHOT_PREFIX}_${sanitizeString(errorStackHash)}_${sanitizeString(errorMessagePrefix)}`
.replace(/\W+/g, '-') // Replace non-word characters with a dash
.slice(0, MAX_FILENAME_LENGTH);
return fileName;
}
}