Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: close the page of a timed-out adaptive browser attempt right away (#4371)
Nikita Chapovskii80ec50162041
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 551f93801..d2d1cc7e6 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -705,16 +705,32 @@ export class AdaptivePlaywrightCrawler<Object.defineProperty(subCrawlerContext, key, { ...descriptor, configurable: false });}+ // The timeout below does not stop the request handler, and the pipeline closes the page only after the handler+ // settles. Racing the handler against this lets a timed-out attempt release its page right away.+ const attemptFailed = new AbortController();+try {// Any failure - middleware or handler - ends up in the `ok: false` branch below.const rethrow = (error: unknown) => {throw error;};+ const abandonableRequestHandler = async (crawlingContext: ExtendedContext) => {+ // The attempt may have failed before the handler starts, and a late abort listener would never fire.+ attemptFailed.signal.throwIfAborted();+ return Promise.race([+ this.requestHandler(crawlingContext),+ new Promise<never>((_resolve, reject) => {+ attemptFailed.signal.addEventListener('abort', () => reject(attemptFailed.signal.reason), {+ once: true,+ });+ }),+ ]);+ };const callAdaptiveRequestHandler = async () => {if (renderingType === 'static') {await this.#staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this), rethrow);} else if (renderingType === 'clientOnly') {- await this.#browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this), rethrow);+ await this.#browserContextPipeline.call(subCrawlerContext, abandonableRequestHandler, rethrow);}};@@ -734,6 +750,7 @@ export class AdaptivePlaywrightCrawler<return { ok: true, transaction, contextModifications, logs };} catch (error) {+ attemptFailed.abort(error);return { ok: false, error, transaction, contextModifications, logs };}}diff --git a/test/core/crawlers/adaptive-playwright-crawler.test.ts b/test/core/crawlers/adaptive-playwright-crawler.test.tsindex 2e40f88c0..257dcae8f 100644--- a/test/core/crawlers/adaptive-playwright-crawler.test.ts+++ b/test/core/crawlers/adaptive-playwright-crawler.test.ts@@ -1283,6 +1283,37 @@ describe('AdaptivePlaywrightCrawler', () => {expect(pageClosedInCleanup).toBe(false);});+ test('closes the page of a timed-out browser attempt without waiting for its request handler', async () => {+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({+ detectionProbabilityRecommendation: 0,+ renderingType: 'clientOnly',+ });++ let pageClosed: Promise<boolean> | undefined;+ let pageClosedOnTimeout: boolean | undefined;++ const crawler = await makeOneshotCrawler(+ {+ renderingTypePredictor,+ requestHandlerTimeoutSecs: 3,+ requestHandler: async ({ page }) => {+ pageClosed = new Promise((resolve) => {+ page.once('close', () => resolve(true));+ });+ await pageClosed;+ },+ failedRequestHandler: async () => {+ pageClosedOnTimeout = await Promise.race([pageClosed, sleep(1000).then(() => false)]);+ },+ },+ [`http://${HOSTNAME}:${port}/static`],+ );++ await crawler.run();++ expect(pageClosedOnTimeout).toBe(true);+ });+test('should persist RenderingTypePredictor state on PERSIST_STATE events', async () => {const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async ({ pushData }) => {await pushData({ content: 'test data' });