Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding tests for resource blocking
Marek Trunkatbcea729065bf
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex 73c72b666..cc52dc61a 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -151,30 +151,51 @@ describe('Apify.utils.puppeteer', () => {it('supports blockResources() with default values', async () => {const browser = await Apify.launchPuppeteer({ headless: true });+ const loadedUrls = [];try {const page = await browser.newPage();await Apify.utils.puppeteer.blockResources(page);- await page.goto('about:blank');-- // TODO: Write some proper unit test for this+ page.on('response', response => loadedUrls.push(response.url()));+ await page.setContent(`<html>+ <body>+ <link rel="stylesheet" type="text/css" href="https://example.com/style.css">+ <img src="https://example.com/image.png" />+ <script src="https://example.com/script.js" defer="defer">></script>+ </body>+ </html>`, { waitUntil: 'networkidle0' });} finally {- browser.close();+ await browser.close();}++ expect(loadedUrls).to.be.eql([+ 'https://example.com/script.js',+ ]);});it('supports blockResources() with nondefault values', async () => {const browser = await Apify.launchPuppeteer({ headless: true });+ const loadedUrls = [];try {const page = await browser.newPage();- await Apify.utils.puppeteer.blockResources(page, ['font']);- await page.goto('about:blank');-- // TODO: Write some proper unit test for this+ await Apify.utils.puppeteer.blockResources(page, ['script']);+ page.on('response', response => loadedUrls.push(response.url()));+ await page.setContent(`<html>+ <body>+ <link rel="stylesheet" type="text/css" href="https://example.com/style.css">+ <img src="https://example.com/image.png" />+ <script src="https://example.com/script.js" defer="defer">></script>+ </body>+ </html>`, { waitUntil: 'networkidle0' });} finally {- browser.close();+ await browser.close();}++ expect(loadedUrls).to.be.eql([+ 'https://example.com/style.css',+ 'https://example.com/image.png',+ ]);});it('supports cacheResponses()', async () => {