Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: Make response and enqueueLinks in adaptive crawler context consistent (#4279)
Jan Buchara2029cdcb15c
diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex c1cfcb26c..3e0a2b315 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -54,6 +54,7 @@"@crawlee/browser-pool": "workspace:*","@crawlee/cheerio": "workspace:*","@crawlee/core": "workspace:*",+ "@crawlee/http-client": "workspace:*","@crawlee/types": "workspace:*","@crawlee/utils": "workspace:*","cheerio": "^1.0.0",diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex 336828cdb..afec12bca 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -31,6 +31,7 @@ import {import { extractUrlsFromPage } from '@crawlee/browser';import type { CheerioCrawlingContext } from '@crawlee/cheerio';import { CheerioCrawler } from '@crawlee/cheerio';+import { ResponseWithUrl } from '@crawlee/http-client';import type {AddRequestsBatchedResult,CrawleeLogger,@@ -567,7 +568,9 @@ export class AdaptivePlaywrightCrawler<);return (await this.#enqueueLinks(urls, options, cheerioContext.request)) as unknown as void;},- response: cheerioContext.response,+ // The HTTP crawler has already consumed the body; hand out a fresh copy, like the browser side does.+ // HTML bodies are already decoded to UTF-8, so `arrayBuffer()` may disagree with the original headers.+ response: new ResponseWithUrl(cheerioContext.body, cheerioContext.response),};}@@ -576,16 +579,20 @@ export class AdaptivePlaywrightCrawler<const originalResponse = playwrightContext.response;return {- response: new Response(Uint8Array.from(await originalResponse.body()), {+ response: new ResponseWithUrl(Uint8Array.from(await originalResponse.body()), {headers: originalResponse.headers(),status: originalResponse.status(),statusText: originalResponse.statusText(),+ url: originalResponse.url(),}),enqueueLinks: async (options: EnqueueLinksOptions = {}, timeoutMs = 5000) => {// TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplicationconst selector = options.selector ?? 'a';const locator = playwrightContext.page.locator(selector).first();- await locator.waitFor({ timeout: timeoutMs, state: 'attached' });+ // Wait for client-rendered links, but like the HTTP side, enqueue nothing when there are none.+ await locator.waitFor({ timeout: timeoutMs, state: 'attached' }).catch((error: Error) => {+ if (error.name !== 'TimeoutError') throw error;+ });const urls = await extractUrlsFromPage(playwrightContext.page,selector,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 1b3403b3d..42751a9e5 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -890,6 +890,9 @@ importers:'@crawlee/core':specifier: workspace:*version: link:../core+ '@crawlee/http-client':+ specifier: workspace:*+ version: link:../http-client'@crawlee/types':specifier: workspace:*version: link:../typesdiff --git a/test/core/crawlers/adaptive-playwright-crawler.test.ts b/test/core/crawlers/adaptive-playwright-crawler.test.tsindex f03fc1e7e..842ce36f8 100644--- a/test/core/crawlers/adaptive-playwright-crawler.test.ts+++ b/test/core/crawlers/adaptive-playwright-crawler.test.ts@@ -674,6 +674,32 @@ describe('AdaptivePlaywrightCrawler', () => {crawlDepth: 1,});});++ test.each([['static'], ['clientOnly']] as const)(+ 'enqueues nothing when the selector matches nothing (%s)',+ async (renderingType) => {+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({+ detectionProbabilityRecommendation: 0,+ renderingType,+ });++ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(+ async ({ enqueueLinks, pushData }) => {+ const { addedRequests } = await enqueueLinks({ selector: '.missing' });+ await pushData({ enqueued: addedRequests.length });+ },+ );++ const crawler = await makeOneshotCrawler({ requestHandler, renderingTypePredictor }, [+ `http://${HOSTNAME}:${port}/static`,+ ]);++ await crawler.run();++ expect(requestHandler).toHaveBeenCalledOnce();+ expect((await Dataset.getData()).items).toEqual([{ enqueued: 0 }]);+ },+ );});test.each([['static'], ['clientOnly']] as const)(@@ -720,6 +746,27 @@ describe('AdaptivePlaywrightCrawler', () => {},);+ test.each([['static'], ['clientOnly']] as const)('response body is readable (%s)', async (renderingType) => {+ const renderingTypePredictor = makeRiggedRenderingTypePredictor({+ detectionProbabilityRecommendation: 0,+ renderingType,+ });+ const url = `http://${HOSTNAME}:${port}/static`;++ const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(+ async ({ response, pushData }) => {+ await pushData({ url: response.url, hasHeading: (await response.text()).includes('<h1>Heading</h1>') });+ },+ );++ const crawler = await makeOneshotCrawler({ requestHandler, renderingTypePredictor }, [url]);++ await crawler.run();++ expect(requestHandler).toHaveBeenCalledOnce();+ expect((await Dataset.getData()).items).toEqual([{ url, hasHeading: true }]);+ });+test('should persist crawler state', async () => {const renderingTypePredictor = makeRiggedRenderingTypePredictor({detectionProbabilityRecommendation: 0,