Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding request into handlePageFunction args
Marek Trunkatf20e12ee2a71
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex bfaaed237..3dc6c3047 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -48,7 +48,8 @@ const DEFAULT_OPTIONS = {* @param {RequestList} options.requestList List of the requests to be processed. (See `requestList` parameter of `Apify.BasicCrawler`)* @param {Function} options.handlePageFunction Function to process each request.* @param {Number} [options.pageOpsTimeoutMillis=30000] Timeout for options.handlePagefunction- * @param {Function} [options.gotoFunction=({ request, page }) => page.goto(request.url)] Overrides default gotoFunction.+ * @param {Function} [options.gotoFunction=({ request, page }) => page.goto(request.url)] Overrides default gotoFunction. This function+ * should return a result of page.goto(), ie. Puppeteers Response object.* @param {Function} [options.handleFailedRequestFunction=({ request }) => log.error('Request failed', _.pick(request, 'url', 'uniqueKey'))]* Function to handle requests that failed more then option.maxRequestRetries times. (See `handleFailedRequestFunction`@@ -166,8 +167,13 @@ export default class PuppeteerCrawler {.newPage().then((newPage) => { page = newPage; }).then(() => this.gotoFunction({ page, request, puppeteerPool: this.puppeteerPool }))- .then(() => {- const promise = this.handlePageFunction({ page, request, puppeteerPool: this.puppeteerPool });+ .then((response) => {+ const promise = this.handlePageFunction({+ page,+ request,+ puppeteerPool: this.puppeteerPool,+ response,+ });if (!isPromise(promise)) throw new Error('User provided handlePageFunction must return a Promise.');diff --git a/test/puppeteer_crawler.js b/test/puppeteer_crawler.jsindex 9c389450c..e80d47253 100644--- a/test/puppeteer_crawler.js+++ b/test/puppeteer_crawler.js@@ -31,11 +31,11 @@ describe('puppeteer_crawler', () => {const processed = [];const failed = [];const requestList = new Apify.RequestList({ sources });- const handlePageFunction = async ({ page, request }) => {+ const handlePageFunction = async ({ page, request, response }) => {await page.waitFor('title');+ expect(await response.status()).to.be.eql(200);request.userData.title = await page.title();-processed.push(request);};