Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(adaptive-crawler): Unify deferred cleanup order with BasicCrawler (#4328)
Jan Buchar8bdd6b0b9582
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex d1443573a..551f93801 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -525,13 +525,33 @@ export class AdaptivePlaywrightCrawler<this.#staticCrawler = staticCrawler;this.#browserCrawler = browserCrawler;- this.#staticContextPipeline = staticCrawler.contextPipeline.compose(- this.#adaptCheerioContext.bind(this),- ) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;+ // Each attempt needs its own deferred cleanups, and they must share the attempt's cleanup stack: resources the+ // pipeline defers releasing (the page) have to outlive the user's cleanups, which may still use them.+ const deferredCleanup = ContextPipeline.create<CrawlingContext>().compose((_context, onCleanup) => ({+ registerDeferredCleanup: (cleanup: () => Promise<unknown>) => {+ onCleanup(async () => {+ try {+ await cleanup();+ } catch (error) {+ this.log.debug('Error in deferred cleanup', { error });+ }+ });+ },+ }));- this.#browserContextPipeline = browserCrawler.contextPipeline.compose(- this.#adaptPlaywrightContext.bind(this),- ) as unknown as ContextPipeline<CrawlingContext, ExtendedContext>;+ this.#staticContextPipeline = deferredCleanup+ .chain(staticCrawler.contextPipeline)+ .compose(this.#adaptCheerioContext.bind(this)) as unknown as ContextPipeline<+ CrawlingContext,+ ExtendedContext+ >;++ this.#browserContextPipeline = deferredCleanup+ .chain(browserCrawler.contextPipeline)+ .compose(this.#adaptPlaywrightContext.bind(this)) as unknown as ContextPipeline<+ CrawlingContext,+ ExtendedContext+ >;}protected override async init(): Promise<void> {@@ -654,8 +674,6 @@ export class AdaptivePlaywrightCrawler<const logs: LogProxyCall[] = [];- const deferredCleanup: (() => Promise<unknown>)[] = [];-const contextModifications: ContextModifications = {request: new MutationJournal(),state: new MutationJournal(),@@ -671,7 +689,6 @@ export class AdaptivePlaywrightCrawler<useState: async (defaultValue?: Dictionary) =>contextModifications.state.track(await useStateFunction(defaultValue)),log: this.#createLogProxy(context.log, logs),- registerDeferredCleanup: (cleanup: () => Promise<unknown>) => deferredCleanup.push(cleanup),// The sub-crawler pipeline keeps this attempt's navigation deadline on `subCrawlerContext`, but the// outer `extendTimeout` only pushes the deadline it finds on `context`, so push this one as well.extendTimeout: (secs: number) => {@@ -718,8 +735,6 @@ export class AdaptivePlaywrightCrawler<return { ok: true, transaction, contextModifications, logs };} catch (error) {return { ok: false, error, transaction, contextModifications, logs };- } finally {- await Promise.all(deferredCleanup.map((cleanup) => cleanup()));}}diff --git a/test/core/crawlers/adaptive-playwright-crawler.test.ts b/test/core/crawlers/adaptive-playwright-crawler.test.tsindex 848354d49..cf7f7bcf8 100644--- a/test/core/crawlers/adaptive-playwright-crawler.test.ts+++ b/test/core/crawlers/adaptive-playwright-crawler.test.ts@@ -1216,6 +1216,73 @@ describe('AdaptivePlaywrightCrawler', () => {expect((await Dataset.getData()).items).toEqual([{ run: 1 }]);});+ test('runs deferred cleanups one at a time, newest first, without failing the request on a cleanup error', async () => {+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({+ detectionProbabilityRecommendation: 0,+ renderingType: 'static',+ });++ const events: string[] = [];+ const failedRequestHandler = vi.fn();++ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = async ({+ pushData,+ registerDeferredCleanup,+ }) => {+ await pushData({ ok: true });+ registerDeferredCleanup(async () => {+ events.push('first');+ });+ registerDeferredCleanup(async () => {+ events.push('second');+ throw new Error('cleanup failed');+ });+ // Yields mid-cleanup, so concurrent execution would interleave the others between start and end.+ registerDeferredCleanup(async () => {+ events.push('third:start');+ await Promise.resolve();+ events.push('third:end');+ });+ };++ const crawler = await makeOneshotCrawler(+ { requestHandler, renderingTypePredictor, maxRequestRetries: 3, failedRequestHandler },+ [`http://${HOSTNAME}:${port}/static`],+ );++ await crawler.run();++ expect(events).toEqual(['third:start', 'third:end', 'second', 'first']);+ expect(failedRequestHandler).not.toHaveBeenCalled();+ expect((await Dataset.getData()).items).toEqual([{ ok: true }]);+ });++ test('closes the page only after the deferred cleanups of a browser attempt', async () => {+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({+ detectionProbabilityRecommendation: 0,+ renderingType: 'clientOnly',+ });++ let pageClosedInCleanup: boolean | undefined;++ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = async ({+ page,+ registerDeferredCleanup,+ }) => {+ registerDeferredCleanup(async () => {+ pageClosedInCleanup = page.isClosed();+ });+ };++ const crawler = await makeOneshotCrawler({ requestHandler, renderingTypePredictor }, [+ `http://${HOSTNAME}:${port}/dynamic`,+ ]);++ await crawler.run();++ expect(pageClosedInCleanup).toBe(false);+ });+test('should persist RenderingTypePredictor state on PERSIST_STATE events', async () => {const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async ({ pushData }) => {await pushData({ content: 'test data' });