Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Update PR with comments
Ondra Urbana89a866e63bc
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex 729211e6d..2e9ad1b41 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -309,6 +309,9 @@ class BasicCrawler {const request = await this._fetchNextRequest();if (!request) return;+ // Reset loadedUrl so an old one is not carried over to retries.+ request.loadedUrl = null;+try {await this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool });await source.markRequestHandled(request);diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex ce3d1d037..c19567344 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -292,10 +292,6 @@ class PuppeteerCrawler {async _handleRequestFunction({ request, autoscaledPool }) {const page = await this.puppeteerPool.newPage();try {- // Delete loadedUrl before navigation to prevent user- // from getting an old loadedUrl in his custom gotoFunction- request.loadedUrl = null;-const response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool });request.loadedUrl = page.url();await addTimeoutToPromise(diff --git a/src/utils.js b/src/utils.jsindex 6681a228f..f2b8aef58 100644--- a/src/utils.js+++ b/src/utils.js@@ -595,6 +595,7 @@ const createRequestDebugInfo = (request, response = {}, additionalFields = {}) ={requestId: request.id,url: request.url,+ loadedUrl: request.loadedUrl,method: request.method,retryCount: request.retryCount,errorMessages: request.errorMessages,diff --git a/test/autoscaling/snapshotter.js b/test/autoscaling/snapshotter.jsindex 0db7e98a7..babd68ded 100644--- a/test/autoscaling/snapshotter.js+++ b/test/autoscaling/snapshotter.js@@ -126,30 +126,30 @@ describe('Snapshotter', () => {});it('correctly marks eventLoopOverloaded', () => { /* eslint-disable no-underscore-dangle */- const noop = () => {};- const block = (millis) => {- const start = Date.now();- while (start + millis > Date.now()) { /* empty */ }- };-- const snapshotter = new Snapshotter({ maxBlockedMillis: 5, eventLoopSnapshotIntervalSecs: 0 });- snapshotter._snapshotEventLoop(noop);- block(1);- snapshotter._snapshotEventLoop(noop);- block(2);- snapshotter._snapshotEventLoop(noop);- block(7);- snapshotter._snapshotEventLoop(noop);- block(3);- snapshotter._snapshotEventLoop(noop);- const loopSnapshots = snapshotter.getEventLoopSample();-- expect(loopSnapshots.length).to.be.eql(5);- expect(loopSnapshots[0].isOverloaded).to.be.eql(false);- expect(loopSnapshots[1].isOverloaded).to.be.eql(false);- expect(loopSnapshots[2].isOverloaded).to.be.eql(false);- expect(loopSnapshots[3].isOverloaded).to.be.eql(true);- expect(loopSnapshots[4].isOverloaded).to.be.eql(false);+ const clock = sinon.useFakeTimers();+ try {+ const noop = () => {};+ const snapshotter = new Snapshotter({ maxBlockedMillis: 5, eventLoopSnapshotIntervalSecs: 0 });+ snapshotter._snapshotEventLoop(noop);+ clock.tick(1);+ snapshotter._snapshotEventLoop(noop);+ clock.tick(2);+ snapshotter._snapshotEventLoop(noop);+ clock.tick(7);+ snapshotter._snapshotEventLoop(noop);+ clock.tick(3);+ snapshotter._snapshotEventLoop(noop);+ const loopSnapshots = snapshotter.getEventLoopSample();++ expect(loopSnapshots.length).to.be.eql(5);+ expect(loopSnapshots[0].isOverloaded).to.be.eql(false);+ expect(loopSnapshots[1].isOverloaded).to.be.eql(false);+ expect(loopSnapshots[2].isOverloaded).to.be.eql(false);+ expect(loopSnapshots[3].isOverloaded).to.be.eql(true);+ expect(loopSnapshots[4].isOverloaded).to.be.eql(false);+ } finally {+ clock.restore();+ }});it('correctly marks memoryOverloaded', async () => { /* eslint-disable no-underscore-dangle */diff --git a/test/utils.js b/test/utils.jsindex c02befcfb..3d8c3e8b0 100644--- a/test/utils.js+++ b/test/utils.js@@ -792,6 +792,7 @@ describe('utils.createRequestDebugInfo()', () => {const request = {id: 'some-id',url: 'https://example.com',+ loadedUrl: 'https://example.com',method: 'POST',retryCount: 2,errorMessages: ['xxx'],@@ -811,6 +812,7 @@ describe('utils.createRequestDebugInfo()', () => {expect(Apify.utils.createRequestDebugInfo(request, response, additionalFields)).to.be.eql({requestId: 'some-id',url: 'https://example.com',+ loadedUrl: 'https://example.com',method: 'POST',retryCount: 2,errorMessages: ['xxx'],@@ -823,6 +825,7 @@ describe('utils.createRequestDebugInfo()', () => {const request = {id: 'some-id',url: 'https://example.com',+ loadedUrl: 'https://example.com',method: 'POST',retryCount: 2,errorMessages: ['xxx'],@@ -842,6 +845,7 @@ describe('utils.createRequestDebugInfo()', () => {expect(Apify.utils.createRequestDebugInfo(request, response, additionalFields)).to.be.eql({requestId: 'some-id',url: 'https://example.com',+ loadedUrl: 'https://example.com',method: 'POST',retryCount: 2,errorMessages: ['xxx'],