Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Lint fix
Ondra Urbanfa40be15f5ae
diff --git a/test/actor.test.js b/test/actor.test.jsindex 117237243..407a5f2fe 100644--- a/test/actor.test.js+++ b/test/actor.test.js@@ -219,7 +219,7 @@ describe('Apify.main()', () => {},exitCode: 91,});- }+ },);test(@@ -232,7 +232,7 @@ describe('Apify.main()', () => {},exitCode: 91,});- }+ },);});@@ -1068,7 +1068,7 @@ describe('Apify.getApifyProxyUrl()', () => {delete process.env[ENV_VARS.PROXY_PORT];expect(Apify.getApifyProxyUrl({ password: 'xyz' })).toEqual(- `http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`+ `http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`,);expect(() => Apify.getApifyProxyUrl()).toThrowError();@@ -1108,7 +1108,7 @@ describe('Apify.getApifyProxyUrl()', () => {delete process.env[ENV_VARS.PROXY_PORT];expect(Apify.getApifyProxyUrl({ password: 'xyz' })).toEqual(- `http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`+ `http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`,);expect(() => Apify.getApifyProxyUrl()).toThrowError();diff --git a/test/autoscaling/autoscaled_pool.test.js b/test/autoscaling/autoscaled_pool.test.jsindex c22ef01b5..0f55d4b4d 100644--- a/test/autoscaling/autoscaled_pool.test.js+++ b/test/autoscaling/autoscaled_pool.test.js@@ -349,7 +349,7 @@ describe('AutoscaledPool', () => {expect(count).toBeGreaterThanOrEqual(11);// Check finished tasks.expect(count).toBeLessThanOrEqual(13);- }+ },);test(@@ -378,7 +378,7 @@ describe('AutoscaledPool', () => {// Check finished tasks.expect(finished).toHaveLength(20);expect(finished[11] - finished[10]).toBeGreaterThan(9);- }+ },);test('should work with loggingIntervalMillis = null', async () => {diff --git a/test/autoscaling/snapshotter.test.js b/test/autoscaling/snapshotter.test.jsindex 8b41000e2..83492db77 100644--- a/test/autoscaling/snapshotter.test.js+++ b/test/autoscaling/snapshotter.test.js@@ -232,7 +232,7 @@ describe('Snapshotter', () => {expect(memorySnapshots[4].isOverloaded).toBe(false);delete process.env[ENV_VARS.MEMORY_MBYTES];- }+ },);test('correctly logs critical memory overload', () => {@@ -287,7 +287,7 @@ describe('Snapshotter', () => {expect(clientSnapshots[3].isOverloaded).toBe(true);utils.apifyClient.stats = oldStats;- }+ },);test('.get[.*]Sample limits amount of samples', async () => {diff --git a/test/crawlers/basic_crawler.test.js b/test/crawlers/basic_crawler.test.jsindex 80078c76b..58aa2c44a 100644--- a/test/crawlers/basic_crawler.test.js+++ b/test/crawlers/basic_crawler.test.js@@ -92,7 +92,7 @@ describe('BasicCrawler', () => {// clean upawait basicCrawler.autoscaledPool._destroy(); // eslint-disable-line no-underscore-dangle- }+ },);test('should retry failed requests', async () => {@@ -400,7 +400,7 @@ describe('BasicCrawler', () => {});expect(await crawler._isTaskReadyFunction()).toBe(false); // eslint-disable-line no-underscore-dangle- }+ },);test(@@ -450,7 +450,7 @@ describe('BasicCrawler', () => {mock.verify();sinon.restore();- }+ },);test('should support maxRequestsPerCrawl parameter', async () => {diff --git a/test/enqueue_links/click_elements.test.js b/test/enqueue_links/click_elements.test.jsindex 7cbbd6ea3..19e843d93 100644--- a/test/enqueue_links/click_elements.test.js+++ b/test/enqueue_links/click_elements.test.js@@ -199,7 +199,7 @@ describe('enqueueLinksByClickingElements()', () => {expect(interceptedRequests[0].url).toBe('https://example.com/#foo');const pageContent = await page.content();expect(pageContent).toMatch('onclick="return window.location = ');- }+ },);test('should prevent reload from cache with window.reload()', async () => {diff --git a/test/puppeteer_request_interception.test.js b/test/puppeteer_request_interception.test.jsindex 6a9f661d2..1ee347f02 100644--- a/test/puppeteer_request_interception.test.js+++ b/test/puppeteer_request_interception.test.js@@ -94,7 +94,7 @@ describe('Apify.utils.puppeteer.addInterceptRequestHandler|removeInterceptRequesexpect(propagatedUrls).toEqual(expect.arrayContaining(['https://example.com/style.css',]));- }+ },);test('should allow to modify request', async () => {diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.jsindex 7bdaacb41..8d625abe8 100644--- a/test/session_pool/session.test.js+++ b/test/session_pool/session.test.js@@ -36,7 +36,7 @@ describe('Session - testing session behaviour ', () => {}expect(err).toBeDefined(); // eslint-disable-lineexpect(err.message.includes('Session: sessionPool must be instance of SessionPool')).toBe(true); // eslint-disable-line- }+ },);test('should mark session markBaded', () => {diff --git a/test/session_pool/session_pool.test.js b/test/session_pool/session_pool.test.jsindex ba04c4304..f2f2ebbfd 100644--- a/test/session_pool/session_pool.test.js+++ b/test/session_pool/session_pool.test.js@@ -115,7 +115,7 @@ describe('SessionPool - testing session pool', () => {expect(isCalled).toBe(true); //eslint-disable-lineexpect(sessionPool.sessions[0].id === session.id).toBe(false); //eslint-disable-lineexpect(sessionPool.sessions).toHaveLength(1);- }+ },);});diff --git a/test/stealth/stealth.test.js b/test/stealth/stealth.test.jsindex dd248129c..0bfac0366 100644--- a/test/stealth/stealth.test.js+++ b/test/stealth/stealth.test.js@@ -240,7 +240,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {expect(failedChecks.length).toBe(0);return browser.close();- }+ },);test('should work in crawler', async () => {diff --git a/test/utils.test.js b/test/utils.test.jsindex 33976b609..861cb44c7 100644--- a/test/utils.test.js+++ b/test/utils.test.js@@ -122,7 +122,7 @@ describe('utils.isDocker()', () => {});describe('utils.getMemoryInfo()', () => {- test('works WITHOUT child process outside the container', () => {+ test('works WITHOUT child process outside the container', async () => {const osMock = sinon.mock(os);const utilsMock = sinon.mock(utils);@@ -141,23 +141,22 @@ describe('utils.getMemoryInfo()', () => {.once().returns(333);- return Apify- .getMemoryInfo()- .then((data) => {- expect(data).toMatchObject({- totalBytes: 333,- freeBytes: 222,- usedBytes: 111,- childProcessesBytes: 0,- });- expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);-- utilsMock.restore();- osMock.restore();+ try {+ const data = await Apify.getMemoryInfo();+ expect(data).toMatchObject({+ totalBytes: 333,+ freeBytes: 222,+ usedBytes: 111,+ childProcessesBytes: 0,});+ expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);+ } finally {+ utilsMock.verify();+ osMock.verify();+ }});- test('works WITHOUT child process inside the container', () => {+ test('works WITHOUT child process inside the container', async () => {const utilsMock = sinon.mock(utils);utilsMock@@ -173,23 +172,22 @@ describe('utils.getMemoryInfo()', () => {else throw new Error('Invalid path');});- return Apify- .getMemoryInfo()- .then((data) => {- expect(data).toMatchObject({- totalBytes: 333,- freeBytes: 222,- usedBytes: 111,- childProcessesBytes: 0,- });- expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);-- utilsMock.restore();- fs.readFile.restore();+ try {+ const data = await Apify.getMemoryInfo();+ expect(data).toMatchObject({+ totalBytes: 333,+ freeBytes: 222,+ usedBytes: 111,+ childProcessesBytes: 0,});+ expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);+ } finally {+ utilsMock.verify();+ fs.readFile.restore();+ }});- test('works WITH child process outside the container', () => {+ test('works WITH child process outside the container', async () => {const osMock = sinon.mock(os);const utilsMock = sinon.mock(utils);process.env[ENV_VARS.HEADLESS] = '1';@@ -209,27 +207,26 @@ describe('utils.getMemoryInfo()', () => {.once().returns(333);- return Apify.launchPuppeteer()- .then((browser) => {- return Apify- .getMemoryInfo()- .then((data) => {- expect(data).toMatchObject({- totalBytes: 333,- freeBytes: 222,- usedBytes: 111,- });- expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);- expect(data.childProcessesBytes).toBeGreaterThanOrEqual(20000000);- utilsMock.restore();- osMock.restore();- delete process.env[ENV_VARS.HEADLESS];- })- .then(() => browser.close());+ let browser;+ try {+ browser = await Apify.launchPuppeteer();+ const data = await Apify.getMemoryInfo();+ expect(data).toMatchObject({+ totalBytes: 333,+ freeBytes: 222,+ usedBytes: 111,});+ expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);+ expect(data.childProcessesBytes).toBeGreaterThanOrEqual(20000000);+ } finally {+ utilsMock.verify();+ osMock.verify();+ delete process.env[ENV_VARS.HEADLESS];+ if (browser) await browser.close();+ }});- test('works WITH child process inside the container', () => {+ test('works WITH child process inside the container', async () => {const utilsMock = sinon.mock(utils);process.env[ENV_VARS.HEADLESS] = '1';@@ -246,24 +243,23 @@ describe('utils.getMemoryInfo()', () => {else throw new Error('Invalid path');});- return Apify.launchPuppeteer()- .then((browser) => {- return Apify- .getMemoryInfo()- .then((data) => {- expect(data).toMatchObject({- totalBytes: 333,- freeBytes: 222,- usedBytes: 111,- });- expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);- expect(data.childProcessesBytes).toBeGreaterThanOrEqual(20000000);- utilsMock.restore();- fs.readFile.restore();- delete process.env[ENV_VARS.HEADLESS];- })- .then(() => browser.close());+ let browser;+ try {+ browser = await Apify.launchPuppeteer();+ const data = await Apify.getMemoryInfo();+ expect(data).toMatchObject({+ totalBytes: 333,+ freeBytes: 222,+ usedBytes: 111,});+ expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);+ expect(data.childProcessesBytes).toBeGreaterThanOrEqual(20000000);+ } finally {+ utilsMock.verify();+ fs.readFile.restore();+ delete process.env[ENV_VARS.HEADLESS];+ if (browser) browser.close();+ }});});@@ -536,7 +532,7 @@ describe('Apify.utils.getRandomUserAgent()', () => {});});-describe('utils.openLocalStorage()', () => {+describe('utils.openLocalStorage()', () => {test('should return item from cache if available and create new one otherwise',async () => {@@ -556,7 +552,7 @@ describe('utils.openLocalStorage()', () => {const store3 = await utils.openLocalStorage('some-other-id', 'some-env', MyStore, cache);expect(store3).not.toBe(store);expect(cache.length()).toBe(2);- }+ },);test(@@ -575,7 +571,7 @@ describe('utils.openLocalStorage()', () => {expect(store.id).toBe('id-from-env');delete process.env['some-env'];- }+ },);test(@@ -598,11 +594,11 @@ describe('utils.openLocalStorage()', () => {const store = await utils.openLocalStorage(null, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, MyStore, cache);expect(store.id).toEqual(defaultLocalValue);- }+ },);});-describe('utils.openRemoteStorage()', () => {+describe('utils.openRemoteStorage()', () => {test('should return item from cache if available and create new one otherwise',async () => {@@ -630,7 +626,7 @@ describe('utils.openRemoteStorage()', () => {expect(store3).not.toBe(store);expect(store3.id).toBe('some-other-id');expect(cache.length()).toBe(2);- }+ },);test(@@ -649,7 +645,7 @@ describe('utils.openRemoteStorage()', () => {expect(store.id).toBe('id-from-env');delete process.env['some-env'];- }+ },);test(@@ -668,7 +664,7 @@ describe('utils.openRemoteStorage()', () => {expect(store.id).toBe('id-from-env');delete process.env['some-env'];- }+ },);});diff --git a/test/utils_social.test.js b/test/utils_social.test.jsindex 62298a9a9..9851ff059 100644--- a/test/utils_social.test.js+++ b/test/utils_social.test.js@@ -516,10 +516,10 @@ describe('utils.social', () => {https://www.linkedin.com/in/someverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongnamelinkedin.com/in/carlnewman`.match(social.LINKEDIN_REGEX_GLOBAL)).toEqual([- 'https://www.linkedin.com/in/bobnewman',- 'http://ie.linkedin.com/in/alicenewman',- 'linkedin.com/in/carlnewman',- ]);+ 'https://www.linkedin.com/in/bobnewman',+ 'http://ie.linkedin.com/in/alicenewman',+ 'linkedin.com/in/carlnewman',+ ]);expect(`-https://www.linkedin.com/in/bobnewman/sub-dir:http://ie.linkedin.com/in/alicenewman?param=1@@ -527,9 +527,9 @@ describe('utils.social', () => {alinkedin.com/in/carlnewman_linkedin.com/in/carlnewman`.match(social.LINKEDIN_REGEX_GLOBAL)).toEqual([- 'https://www.linkedin.com/in/bobnewman/',- 'http://ie.linkedin.com/in/alicenewman',- ]);+ 'https://www.linkedin.com/in/bobnewman/',+ 'http://ie.linkedin.com/in/alicenewman',+ ]);expect(''.match(social.LINKEDIN_REGEX_GLOBAL)).toBe(null);});});@@ -582,10 +582,10 @@ describe('utils.social', () => {"instagram.com/old_brno"http://instagr.am/old_plzen`.match(social.INSTAGRAM_REGEX_GLOBAL)).toEqual([- 'https://www.instagram.com/old_prague',- 'instagram.com/old_brno',- 'http://instagr.am/old_plzen',- ]);+ 'https://www.instagram.com/old_prague',+ 'instagram.com/old_brno',+ 'http://instagr.am/old_plzen',+ ]);expect(`-https://www.instagram.com/old_prague/sub-dirinstagr.am/old_plzen?param=1@@ -593,9 +593,9 @@ describe('utils.social', () => {ainstagram.com/old_brno_instagram.com/old_brno`.match(social.INSTAGRAM_REGEX_GLOBAL)).toEqual([- 'https://www.instagram.com/old_prague/',- 'instagr.am/old_plzen',- ]);+ 'https://www.instagram.com/old_prague/',+ 'instagr.am/old_plzen',+ ]);expect(''.match(social.INSTAGRAM_REGEX_GLOBAL)).toBe(null);});});@@ -647,10 +647,10 @@ describe('utils.social', () => {www.twitter.com/invalidverylongtwitterhandlenotgoodtwitter.com/bob123?param=1`.match(social.TWITTER_REGEX_GLOBAL)).toEqual([- 'https://www.twitter.com/apify',- 'www.twitter.com/jack/',- 'twitter.com/bob123',- ]);+ 'https://www.twitter.com/apify',+ 'www.twitter.com/jack/',+ 'twitter.com/bob123',+ ]);expect(`-https://www.twitter.com/apifytwitter.com/jack@@ -659,10 +659,10 @@ describe('utils.social', () => {atwitter.com/bob_twitter.com/bob`.match(social.TWITTER_REGEX_GLOBAL)).toEqual([- 'https://www.twitter.com/apify',- 'twitter.com/jack',- 'twitter.com/carl123',- ]);+ 'https://www.twitter.com/apify',+ 'twitter.com/jack',+ 'twitter.com/carl123',+ ]);expect(''.match(social.TWITTER_REGEX_GLOBAL)).toBe(null);});});@@ -725,10 +725,10 @@ describe('utils.social', () => {https://www.facebook.com/waytoolongusernamewaytoolongusernamewaytoolongusernamewaytoolongusernamewaytoolongusernamefb.com/bob123`.match(social.FACEBOOK_REGEX_GLOBAL)).toEqual([- 'https://www.facebook.com/someusername',- 'www.facebook.com/another123/',- 'fb.com/bob123',- ]);+ 'https://www.facebook.com/someusername',+ 'www.facebook.com/another123/',+ 'fb.com/bob123',+ ]);expect(`-https://www.facebook.com/someusername/facebook.com/jack4567@@ -737,10 +737,10 @@ describe('utils.social', () => {afacebook.com/bob_facebook.com/bob`.match(social.FACEBOOK_REGEX_GLOBAL)).toEqual([- 'https://www.facebook.com/someusername/',- 'facebook.com/jack4567',- 'fb.com/carl123',- ]);+ 'https://www.facebook.com/someusername/',+ 'facebook.com/jack4567',+ 'fb.com/carl123',+ ]);expect(''.match(social.FACEBOOK_REGEX_GLOBAL)).toBe(null);});});