Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: BrowserCrawler closes ctx.page before errorHandler runs on navig… (#3655)
moaedyab348864b31e
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 41a8adcb5..9468229c1 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -686,16 +686,18 @@ export abstract class BrowserCrawler<}/**- * Marks session bad in case of navigation timeout.+ * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.*/protected async _handleNavigationTimeout(crawlingContext: Context, error: Error): Promise<void> {- const { session } = crawlingContext;+ const { session, page } = crawlingContext;if (error && error.constructor.name === 'TimeoutError') {handleRequestTimeout({ session, errorMessage: error.message });}- await crawlingContext.page.close();+ // Fire-and-forget: no user code will run on this page after a failed navigation.+ // Swallow rejections: the page may already be detached.+ void page.evaluate(() => window.stop()).catch(() => {});}/**diff --git a/packages/browser-pool/src/abstract-classes/browser-plugin.ts b/packages/browser-pool/src/abstract-classes/browser-plugin.tsindex da13dc8c0..e60c41155 100644--- a/packages/browser-pool/src/abstract-classes/browser-plugin.ts+++ b/packages/browser-pool/src/abstract-classes/browser-plugin.ts@@ -42,6 +42,7 @@ export interface CommonBrowser {export interface CommonPage {close(...args: unknown[]): Promise<unknown>;url(): string | Promise<string>;+ evaluate(pageFunction: ((...args: any[]) => unknown) | string, ...args: unknown[]): Promise<unknown>;}export interface BrowserPluginOptions<LibraryOptions> {diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 1b07bd84a..4b49b0138 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -291,6 +291,43 @@ describe('BrowserCrawler', () => {}});+ test.concurrent('errorHandler has open page after non-timeout navigation error', async () => {+ const localStorageEmulator = new MemoryStorageEmulator();+ await localStorageEmulator.init();+ const puppeteerPlugin = new PuppeteerPlugin(puppeteer);++ try {+ const requestList = await RequestList.open({+ sources: [{ url: `${serverAddress}/?q=1` }],+ });++ const pageClosedStates: boolean[] = [];++ const browserCrawler = new (class extends BrowserCrawlerTest {+ protected override async _navigationHandler(): Promise<HTTPResponse | null | undefined> {+ throw new Error('net::ERR_NAME_NOT_RESOLVED');+ }+ })({+ browserPoolOptions: {+ browserPlugins: [puppeteerPlugin],+ },+ requestList,+ requestHandler: async () => {},+ maxRequestRetries: 1,+ errorHandler: async (ctx) => {+ pageClosedStates.push(ctx.page.isClosed());+ },+ });++ await browserCrawler.run();++ expect(pageClosedStates).toHaveLength(1);+ expect(pageClosedStates[0]).toBe(false);+ } finally {+ await localStorageEmulator.destroy();+ }+ });+test.concurrent('should correctly track request.state', async () => {const localStorageEmulator = new MemoryStorageEmulator();await localStorageEmulator.init();