Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Lint fix

Ondra Urbanfa40be15f5ae
diff --git a/test/actor.test.js b/test/actor.test.js
index 117237243..407a5f2fe 100644
--- a/test/actor.test.js
+++ b/test/actor.test.js
@@ -219,7 +219,7 @@ describe('Apify.main()', () => {
},
exitCode: 91,
});
- }
+ },
);
test(
@@ -232,7 +232,7 @@ describe('Apify.main()', () => {
},
exitCode: 91,
});
- }
+ },
);
});
@@ -1068,7 +1068,7 @@ describe('Apify.getApifyProxyUrl()', () => {
delete process.env[ENV_VARS.PROXY_PORT];
expect(Apify.getApifyProxyUrl({ password: 'xyz' })).toEqual(
- `http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`
+ `http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`,
);
expect(() => Apify.getApifyProxyUrl()).toThrowError();
@@ -1108,7 +1108,7 @@ describe('Apify.getApifyProxyUrl()', () => {
delete process.env[ENV_VARS.PROXY_PORT];
expect(Apify.getApifyProxyUrl({ password: 'xyz' })).toEqual(
- `http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`
+ `http://auto:xyz@${LOCAL_ENV_VARS[ENV_VARS.PROXY_HOSTNAME]}:${LOCAL_ENV_VARS[ENV_VARS.PROXY_PORT]}`,
);
expect(() => Apify.getApifyProxyUrl()).toThrowError();
diff --git a/test/autoscaling/autoscaled_pool.test.js b/test/autoscaling/autoscaled_pool.test.js
index c22ef01b5..0f55d4b4d 100644
--- a/test/autoscaling/autoscaled_pool.test.js
+++ b/test/autoscaling/autoscaled_pool.test.js
@@ -349,7 +349,7 @@ describe('AutoscaledPool', () => {
expect(count).toBeGreaterThanOrEqual(11);
// Check finished tasks.
expect(count).toBeLessThanOrEqual(13);
- }
+ },
);
test(
@@ -378,7 +378,7 @@ describe('AutoscaledPool', () => {
// Check finished tasks.
expect(finished).toHaveLength(20);
expect(finished[11] - finished[10]).toBeGreaterThan(9);
- }
+ },
);
test('should work with loggingIntervalMillis = null', async () => {
diff --git a/test/autoscaling/snapshotter.test.js b/test/autoscaling/snapshotter.test.js
index 8b41000e2..83492db77 100644
--- a/test/autoscaling/snapshotter.test.js
+++ b/test/autoscaling/snapshotter.test.js
@@ -232,7 +232,7 @@ describe('Snapshotter', () => {
expect(memorySnapshots[4].isOverloaded).toBe(false);
delete process.env[ENV_VARS.MEMORY_MBYTES];
- }
+ },
);
test('correctly logs critical memory overload', () => {
@@ -287,7 +287,7 @@ describe('Snapshotter', () => {
expect(clientSnapshots[3].isOverloaded).toBe(true);
utils.apifyClient.stats = oldStats;
- }
+ },
);
test('.get[.*]Sample limits amount of samples', async () => {
diff --git a/test/crawlers/basic_crawler.test.js b/test/crawlers/basic_crawler.test.js
index 80078c76b..58aa2c44a 100644
--- a/test/crawlers/basic_crawler.test.js
+++ b/test/crawlers/basic_crawler.test.js
@@ -92,7 +92,7 @@ describe('BasicCrawler', () => {
// clean up
await basicCrawler.autoscaledPool._destroy(); // eslint-disable-line no-underscore-dangle
- }
+ },
);
test('should retry failed requests', async () => {
@@ -400,7 +400,7 @@ describe('BasicCrawler', () => {
});
expect(await crawler._isTaskReadyFunction()).toBe(false); // eslint-disable-line no-underscore-dangle
- }
+ },
);
test(
@@ -450,7 +450,7 @@ describe('BasicCrawler', () => {
mock.verify();
sinon.restore();
- }
+ },
);
test('should support maxRequestsPerCrawl parameter', async () => {
diff --git a/test/enqueue_links/click_elements.test.js b/test/enqueue_links/click_elements.test.js
index 7cbbd6ea3..19e843d93 100644
--- a/test/enqueue_links/click_elements.test.js
+++ b/test/enqueue_links/click_elements.test.js
@@ -199,7 +199,7 @@ describe('enqueueLinksByClickingElements()', () => {
expect(interceptedRequests[0].url).toBe('https://example.com/#foo');
const pageContent = await page.content();
expect(pageContent).toMatch('onclick="return window.location = ');
- }
+ },
);
test('should prevent reload from cache with window.reload()', async () => {
diff --git a/test/puppeteer_request_interception.test.js b/test/puppeteer_request_interception.test.js
index 6a9f661d2..1ee347f02 100644
--- a/test/puppeteer_request_interception.test.js
+++ b/test/puppeteer_request_interception.test.js
@@ -94,7 +94,7 @@ describe('Apify.utils.puppeteer.addInterceptRequestHandler|removeInterceptReques
expect(propagatedUrls).toEqual(expect.arrayContaining([
'https://example.com/style.css',
]));
- }
+ },
);
test('should allow to modify request', async () => {
diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.js
index 7bdaacb41..8d625abe8 100644
--- a/test/session_pool/session.test.js
+++ b/test/session_pool/session.test.js
@@ -36,7 +36,7 @@ describe('Session - testing session behaviour ', () => {
}
expect(err).toBeDefined(); // eslint-disable-line
expect(err.message.includes('Session: sessionPool must be instance of SessionPool')).toBe(true); // eslint-disable-line
- }
+ },
);
test('should mark session markBaded', () => {
diff --git a/test/session_pool/session_pool.test.js b/test/session_pool/session_pool.test.js
index ba04c4304..f2f2ebbfd 100644
--- a/test/session_pool/session_pool.test.js
+++ b/test/session_pool/session_pool.test.js
@@ -115,7 +115,7 @@ describe('SessionPool - testing session pool', () => {
expect(isCalled).toBe(true); //eslint-disable-line
expect(sessionPool.sessions[0].id === session.id).toBe(false); //eslint-disable-line
expect(sessionPool.sessions).toHaveLength(1);
- }
+ },
);
});
diff --git a/test/stealth/stealth.test.js b/test/stealth/stealth.test.js
index dd248129c..0bfac0366 100644
--- a/test/stealth/stealth.test.js
+++ b/test/stealth/stealth.test.js
@@ -240,7 +240,7 @@ describe('Stealth - testing headless chrome hiding tricks', () => {
expect(failedChecks.length).toBe(0);
return browser.close();
- }
+ },
);
test('should work in crawler', async () => {
diff --git a/test/utils.test.js b/test/utils.test.js
index 33976b609..861cb44c7 100644
--- a/test/utils.test.js
+++ b/test/utils.test.js
@@ -122,7 +122,7 @@ describe('utils.isDocker()', () => {
});
describe('utils.getMemoryInfo()', () => {
- test('works WITHOUT child process outside the container', () => {
+ test('works WITHOUT child process outside the container', async () => {
const osMock = sinon.mock(os);
const utilsMock = sinon.mock(utils);
@@ -141,23 +141,22 @@ describe('utils.getMemoryInfo()', () => {
.once()
.returns(333);
- return Apify
- .getMemoryInfo()
- .then((data) => {
- expect(data).toMatchObject({
- totalBytes: 333,
- freeBytes: 222,
- usedBytes: 111,
- childProcessesBytes: 0,
- });
- expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);
-
- utilsMock.restore();
- osMock.restore();
+ try {
+ const data = await Apify.getMemoryInfo();
+ expect(data).toMatchObject({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ childProcessesBytes: 0,
});
+ expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);
+ } finally {
+ utilsMock.verify();
+ osMock.verify();
+ }
});
- test('works WITHOUT child process inside the container', () => {
+ test('works WITHOUT child process inside the container', async () => {
const utilsMock = sinon.mock(utils);
utilsMock
@@ -173,23 +172,22 @@ describe('utils.getMemoryInfo()', () => {
else throw new Error('Invalid path');
});
- return Apify
- .getMemoryInfo()
- .then((data) => {
- expect(data).toMatchObject({
- totalBytes: 333,
- freeBytes: 222,
- usedBytes: 111,
- childProcessesBytes: 0,
- });
- expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);
-
- utilsMock.restore();
- fs.readFile.restore();
+ try {
+ const data = await Apify.getMemoryInfo();
+ expect(data).toMatchObject({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
+ childProcessesBytes: 0,
});
+ expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);
+ } finally {
+ utilsMock.verify();
+ fs.readFile.restore();
+ }
});
- test('works WITH child process outside the container', () => {
+ test('works WITH child process outside the container', async () => {
const osMock = sinon.mock(os);
const utilsMock = sinon.mock(utils);
process.env[ENV_VARS.HEADLESS] = '1';
@@ -209,27 +207,26 @@ describe('utils.getMemoryInfo()', () => {
.once()
.returns(333);
- return Apify.launchPuppeteer()
- .then((browser) => {
- return Apify
- .getMemoryInfo()
- .then((data) => {
- expect(data).toMatchObject({
- totalBytes: 333,
- freeBytes: 222,
- usedBytes: 111,
- });
- expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);
- expect(data.childProcessesBytes).toBeGreaterThanOrEqual(20000000);
- utilsMock.restore();
- osMock.restore();
- delete process.env[ENV_VARS.HEADLESS];
- })
- .then(() => browser.close());
+ let browser;
+ try {
+ browser = await Apify.launchPuppeteer();
+ const data = await Apify.getMemoryInfo();
+ expect(data).toMatchObject({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
});
+ expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);
+ expect(data.childProcessesBytes).toBeGreaterThanOrEqual(20000000);
+ } finally {
+ utilsMock.verify();
+ osMock.verify();
+ delete process.env[ENV_VARS.HEADLESS];
+ if (browser) await browser.close();
+ }
});
- test('works WITH child process inside the container', () => {
+ test('works WITH child process inside the container', async () => {
const utilsMock = sinon.mock(utils);
process.env[ENV_VARS.HEADLESS] = '1';
@@ -246,24 +243,23 @@ describe('utils.getMemoryInfo()', () => {
else throw new Error('Invalid path');
});
- return Apify.launchPuppeteer()
- .then((browser) => {
- return Apify
- .getMemoryInfo()
- .then((data) => {
- expect(data).toMatchObject({
- totalBytes: 333,
- freeBytes: 222,
- usedBytes: 111,
- });
- expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);
- expect(data.childProcessesBytes).toBeGreaterThanOrEqual(20000000);
- utilsMock.restore();
- fs.readFile.restore();
- delete process.env[ENV_VARS.HEADLESS];
- })
- .then(() => browser.close());
+ let browser;
+ try {
+ browser = await Apify.launchPuppeteer();
+ const data = await Apify.getMemoryInfo();
+ expect(data).toMatchObject({
+ totalBytes: 333,
+ freeBytes: 222,
+ usedBytes: 111,
});
+ expect(data.mainProcessBytes).toBeGreaterThanOrEqual(20000000);
+ expect(data.childProcessesBytes).toBeGreaterThanOrEqual(20000000);
+ } finally {
+ utilsMock.verify();
+ fs.readFile.restore();
+ delete process.env[ENV_VARS.HEADLESS];
+ if (browser) browser.close();
+ }
});
});
@@ -536,7 +532,7 @@ describe('Apify.utils.getRandomUserAgent()', () => {
});
});
-describe('utils.openLocalStorage()', () => {
+describe('utils.openLocalStorage()', () => {
test(
'should return item from cache if available and create new one otherwise',
async () => {
@@ -556,7 +552,7 @@ describe('utils.openLocalStorage()', () => {
const store3 = await utils.openLocalStorage('some-other-id', 'some-env', MyStore, cache);
expect(store3).not.toBe(store);
expect(cache.length()).toBe(2);
- }
+ },
);
test(
@@ -575,7 +571,7 @@ describe('utils.openLocalStorage()', () => {
expect(store.id).toBe('id-from-env');
delete process.env['some-env'];
- }
+ },
);
test(
@@ -598,11 +594,11 @@ describe('utils.openLocalStorage()', () => {
const store = await utils.openLocalStorage(null, ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID, MyStore, cache);
expect(store.id).toEqual(defaultLocalValue);
- }
+ },
);
});
-describe('utils.openRemoteStorage()', () => {
+describe('utils.openRemoteStorage()', () => {
test(
'should return item from cache if available and create new one otherwise',
async () => {
@@ -630,7 +626,7 @@ describe('utils.openRemoteStorage()', () => {
expect(store3).not.toBe(store);
expect(store3.id).toBe('some-other-id');
expect(cache.length()).toBe(2);
- }
+ },
);
test(
@@ -649,7 +645,7 @@ describe('utils.openRemoteStorage()', () => {
expect(store.id).toBe('id-from-env');
delete process.env['some-env'];
- }
+ },
);
test(
@@ -668,7 +664,7 @@ describe('utils.openRemoteStorage()', () => {
expect(store.id).toBe('id-from-env');
delete process.env['some-env'];
- }
+ },
);
});
diff --git a/test/utils_social.test.js b/test/utils_social.test.js
index 62298a9a9..9851ff059 100644
--- a/test/utils_social.test.js
+++ b/test/utils_social.test.js
@@ -516,10 +516,10 @@ describe('utils.social', () => {
https://www.linkedin.com/in/someverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongnamesomeverylongname
linkedin.com/in/carlnewman
`.match(social.LINKEDIN_REGEX_GLOBAL)).toEqual([
- 'https://www.linkedin.com/in/bobnewman',
- 'http://ie.linkedin.com/in/alicenewman',
- 'linkedin.com/in/carlnewman',
- ]);
+ 'https://www.linkedin.com/in/bobnewman',
+ 'http://ie.linkedin.com/in/alicenewman',
+ 'linkedin.com/in/carlnewman',
+ ]);
expect(`
-https://www.linkedin.com/in/bobnewman/sub-dir
:http://ie.linkedin.com/in/alicenewman?param=1
@@ -527,9 +527,9 @@ describe('utils.social', () => {
alinkedin.com/in/carlnewman
_linkedin.com/in/carlnewman
`.match(social.LINKEDIN_REGEX_GLOBAL)).toEqual([
- 'https://www.linkedin.com/in/bobnewman/',
- 'http://ie.linkedin.com/in/alicenewman',
- ]);
+ 'https://www.linkedin.com/in/bobnewman/',
+ 'http://ie.linkedin.com/in/alicenewman',
+ ]);
expect(''.match(social.LINKEDIN_REGEX_GLOBAL)).toBe(null);
});
});
@@ -582,10 +582,10 @@ describe('utils.social', () => {
"instagram.com/old_brno"
http://instagr.am/old_plzen
`.match(social.INSTAGRAM_REGEX_GLOBAL)).toEqual([
- 'https://www.instagram.com/old_prague',
- 'instagram.com/old_brno',
- 'http://instagr.am/old_plzen',
- ]);
+ 'https://www.instagram.com/old_prague',
+ 'instagram.com/old_brno',
+ 'http://instagr.am/old_plzen',
+ ]);
expect(`
-https://www.instagram.com/old_prague/sub-dir
instagr.am/old_plzen?param=1
@@ -593,9 +593,9 @@ describe('utils.social', () => {
ainstagram.com/old_brno
_instagram.com/old_brno
`.match(social.INSTAGRAM_REGEX_GLOBAL)).toEqual([
- 'https://www.instagram.com/old_prague/',
- 'instagr.am/old_plzen',
- ]);
+ 'https://www.instagram.com/old_prague/',
+ 'instagr.am/old_plzen',
+ ]);
expect(''.match(social.INSTAGRAM_REGEX_GLOBAL)).toBe(null);
});
});
@@ -647,10 +647,10 @@ describe('utils.social', () => {
www.twitter.com/invalidverylongtwitterhandlenotgood
twitter.com/bob123?param=1
`.match(social.TWITTER_REGEX_GLOBAL)).toEqual([
- 'https://www.twitter.com/apify',
- 'www.twitter.com/jack/',
- 'twitter.com/bob123',
- ]);
+ 'https://www.twitter.com/apify',
+ 'www.twitter.com/jack/',
+ 'twitter.com/bob123',
+ ]);
expect(`
-https://www.twitter.com/apify
twitter.com/jack
@@ -659,10 +659,10 @@ describe('utils.social', () => {
atwitter.com/bob
_twitter.com/bob
`.match(social.TWITTER_REGEX_GLOBAL)).toEqual([
- 'https://www.twitter.com/apify',
- 'twitter.com/jack',
- 'twitter.com/carl123',
- ]);
+ 'https://www.twitter.com/apify',
+ 'twitter.com/jack',
+ 'twitter.com/carl123',
+ ]);
expect(''.match(social.TWITTER_REGEX_GLOBAL)).toBe(null);
});
});
@@ -725,10 +725,10 @@ describe('utils.social', () => {
https://www.facebook.com/waytoolongusernamewaytoolongusernamewaytoolongusernamewaytoolongusernamewaytoolongusername
fb.com/bob123
`.match(social.FACEBOOK_REGEX_GLOBAL)).toEqual([
- 'https://www.facebook.com/someusername',
- 'www.facebook.com/another123/',
- 'fb.com/bob123',
- ]);
+ 'https://www.facebook.com/someusername',
+ 'www.facebook.com/another123/',
+ 'fb.com/bob123',
+ ]);
expect(`
-https://www.facebook.com/someusername/
facebook.com/jack4567
@@ -737,10 +737,10 @@ describe('utils.social', () => {
afacebook.com/bob
_facebook.com/bob
`.match(social.FACEBOOK_REGEX_GLOBAL)).toEqual([
- 'https://www.facebook.com/someusername/',
- 'facebook.com/jack4567',
- 'fb.com/carl123',
- ]);
+ 'https://www.facebook.com/someusername/',
+ 'facebook.com/jack4567',
+ 'fb.com/carl123',
+ ]);
expect(''.match(social.FACEBOOK_REGEX_GLOBAL)).toBe(null);
});
});