Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding tests for resource blocking

Marek Trunkatbcea729065bf
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index 73c72b666..cc52dc61a 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -151,30 +151,51 @@ describe('Apify.utils.puppeteer', () => {
it('supports blockResources() with default values', async () => {
const browser = await Apify.launchPuppeteer({ headless: true });
+ const loadedUrls = [];
try {
const page = await browser.newPage();
await Apify.utils.puppeteer.blockResources(page);
- await page.goto('about:blank');
-
- // TODO: Write some proper unit test for this
+ page.on('response', response => loadedUrls.push(response.url()));
+ await page.setContent(`<html>
+ <body>
+ <link rel="stylesheet" type="text/css" href="https://example.com/style.css">
+ <img src="https://example.com/image.png" />
+ <script src="https://example.com/script.js" defer="defer">></script>
+ </body>
+ </html>`, { waitUntil: 'networkidle0' });
} finally {
- browser.close();
+ await browser.close();
}
+
+ expect(loadedUrls).to.be.eql([
+ 'https://example.com/script.js',
+ ]);
});
it('supports blockResources() with nondefault values', async () => {
const browser = await Apify.launchPuppeteer({ headless: true });
+ const loadedUrls = [];
try {
const page = await browser.newPage();
- await Apify.utils.puppeteer.blockResources(page, ['font']);
- await page.goto('about:blank');
-
- // TODO: Write some proper unit test for this
+ await Apify.utils.puppeteer.blockResources(page, ['script']);
+ page.on('response', response => loadedUrls.push(response.url()));
+ await page.setContent(`<html>
+ <body>
+ <link rel="stylesheet" type="text/css" href="https://example.com/style.css">
+ <img src="https://example.com/image.png" />
+ <script src="https://example.com/script.js" defer="defer">></script>
+ </body>
+ </html>`, { waitUntil: 'networkidle0' });
} finally {
- browser.close();
+ await browser.close();
}
+
+ expect(loadedUrls).to.be.eql([
+ 'https://example.com/style.css',
+ 'https://example.com/image.png',
+ ]);
});
it('supports cacheResponses()', async () => {