Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Attempt to fix flaky tests
Ondra Urband8369d2b2ea9
diff --git a/package.json b/package.jsonindex 05f94b86b..198800bc1 100644--- a/package.json+++ b/package.json@@ -40,7 +40,7 @@"build": "rm -rf ./build && babel src --out-dir build --copy-files","build-docs": "npm run clean && npm run build && node ./website/tools/build_docs.js","build-readme": "node ./tools/build_readme.js",- "test": "npm run build && jest --testTimeout=10000 --coverage --maxWorkers=2",+ "test": "npm run build && jest --testTimeout=10000 --maxWorkers=3","prepare": "npm run build","prepublishOnly": "(test $TRAVIS || (echo \"Publishing is reserved to the Travis CI!\"; exit 1))","clean": "rm -rf build",diff --git a/test/puppeteer_utils.test.js b/test/puppeteer_utils.test.jsindex 563d8b83a..464b96a90 100644--- a/test/puppeteer_utils.test.js+++ b/test/puppeteer_utils.test.js@@ -122,11 +122,18 @@ describe('Apify.utils.puppeteer', () => {}});- test('blockRequests() works with default values', async () => {- const browser = await Apify.launchPuppeteer({ headless: true });- const loadedUrls = [];+ describe('blockRequests()', () => {+ let browser = null;+ beforeAll(async () => {+ browser = await Apify.launchPuppeteer({ headless: true });+ });+ afterAll(async () => {+ await browser.close();+ });++ test('works with default values', async () => {+ const loadedUrls = [];- try {const page = await browser.newPage();await Apify.utils.puppeteer.blockRequests(page);page.on('response', response => loadedUrls.push(response.url()));@@ -135,18 +142,13 @@ describe('Apify.utils.puppeteer', () => {<img src="https://example.com/image.png"><img src="https://example.com/image.gif"><script src="https://example.com/script.js" defer="defer">></script>- </body></html>`, { waitUntil: 'networkidle0' });- } finally {- await browser.close();- }- expect(loadedUrls).toEqual(['https://example.com/script.js']);- });+ </body></html>`, { waitUntil: 'load' });+ expect(loadedUrls).toEqual(['https://example.com/script.js']);+ });- test('blockRequests() works with overridden values', async () => {- const browser = await Apify.launchPuppeteer({ headless: true });- const loadedUrls = [];+ test('works with overridden values', async () => {+ const loadedUrls = [];- try {const page = await browser.newPage();await Apify.utils.puppeteer.blockRequests(page, {urlPatterns: ['.css'],@@ -157,22 +159,17 @@ describe('Apify.utils.puppeteer', () => {<img src="https://example.com/image.png"><img src="https://example.com/image.gif"><script src="https://example.com/script.js" defer="defer">></script>- </body></html>`, { waitUntil: 'networkidle0' });- } finally {- await browser.close();- }- expect(loadedUrls).toEqual(expect.arrayContaining([- 'https://example.com/image.png',- 'https://example.com/script.js',- 'https://example.com/image.gif',- ]));- });+ </body></html>`, { waitUntil: 'load' });+ expect(loadedUrls).toEqual(expect.arrayContaining([+ 'https://example.com/image.png',+ 'https://example.com/script.js',+ 'https://example.com/image.gif',+ ]));+ });- test('supports blockResources() with default values', async () => {- const browser = await Apify.launchPuppeteer({ headless: true });- const loadedUrls = [];+ test('blockResources() supports default values', async () => {+ const loadedUrls = [];- try {const page = await browser.newPage();await Apify.utils.puppeteer.blockResources(page);page.on('response', response => loadedUrls.push(response.url()));@@ -180,21 +177,16 @@ describe('Apify.utils.puppeteer', () => {<link rel="stylesheet" type="text/css" href="https://example.com/style.css"><img src="https://example.com/image.png" /><script src="https://example.com/script.js" defer="defer">></script>- </body></html>`, { waitUntil: 'networkidle0' });- } finally {- await browser.close();- }+ </body></html>`, { waitUntil: 'load' });- expect(loadedUrls).toEqual(expect.arrayContaining([- 'https://example.com/script.js',- ]));- });+ expect(loadedUrls).toEqual(expect.arrayContaining([+ 'https://example.com/script.js',+ ]));+ });- test('supports blockResources() with nondefault values', async () => {- const browser = await Apify.launchPuppeteer({ headless: true });- const loadedUrls = [];+ test('blockResources() supports nondefault values', async () => {+ const loadedUrls = [];- try {const page = await browser.newPage();await Apify.utils.puppeteer.blockResources(page, ['script']);page.on('response', response => loadedUrls.push(response.url()));@@ -202,17 +194,16 @@ describe('Apify.utils.puppeteer', () => {<link rel="stylesheet" type="text/css" href="https://example.com/style.css"><img src="https://example.com/image.png" /><script src="https://example.com/script.js" defer="defer">></script>- </body></html>`, { waitUntil: 'networkidle0' });- } finally {- await browser.close();- }+ </body></html>`, { waitUntil: 'load' });- expect(loadedUrls).toEqual(expect.arrayContaining([- 'https://example.com/style.css',- 'https://example.com/image.png',- ]));+ expect(loadedUrls).toEqual(expect.arrayContaining([+ 'https://example.com/style.css',+ 'https://example.com/image.png',+ ]));+ });});+test('supports cacheResponses()', async () => {const browser = await Apify.launchPuppeteer({ headless: true });const cache = {};