Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Checked page parameter in blockRequests util function Added .gif to default blocking request URL patterns

drobnikjacb0b6f8c188
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 4de3a0a89..80932dc1a 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -16,7 +16,7 @@ const underscorePath = require.resolve('underscore/underscore-min');
const readFilePromised = util.promisify(fs.readFile);
const MAX_INJECT_FILE_CACHE_SIZE = 10;
-const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.woff', '.pdf', '.zip'];
+const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
/**
* Hides certain Puppeteer fingerprints from the page, in order to help avoid detection of the crawler.
@@ -197,7 +197,7 @@ const enqueueRequestsFromClickableElements = async (page, selector, purls, reque
* by default the function blocks URLs that include these patterns:
*
* ```json
- * [".css", ".jpg", ".jpeg", ".png", ".svg", ".woff", ".pdf", ".zip"]
+ * [".css", ".jpg", ".jpeg", ".png", ".svg", ".gif", ".woff", ".pdf", ".zip"]
* ```
*
* The defaults will be concatenated with the patterns you provide in `options.urlPatterns`.
@@ -243,6 +243,7 @@ const blockRequests = async (page, options = {}) => {
includeDefaults = true,
} = options;
+ checkParamOrThrow(page, 'page', 'Object');
checkParamOrThrow(urlPatterns, 'options.urlPatterns', '[String]');
checkParamOrThrow(includeDefaults, 'options.includeDefaults', 'Boolean');
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index 69c57360d..322754c1e 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -159,7 +159,8 @@ describe('Apify.utils.puppeteer', () => {
page.on('response', response => loadedUrls.push(response.url()));
await page.setContent(`<html><body>
<link rel="stylesheet" type="text/css" href="https://example.com/style.css">
- <img src="https://example.com/image.png" />
+ <img src="https://example.com/image.png">
+ <img src="https://example.com/image.gif">
<script src="https://example.com/script.js" defer="defer">></script>
</body></html>`, { waitUntil: 'networkidle0' });
} finally {
@@ -183,7 +184,8 @@ describe('Apify.utils.puppeteer', () => {
page.on('response', response => loadedUrls.push(response.url()));
await page.setContent(`<html><body>
<link rel="stylesheet" type="text/css" href="https://example.com/style.css">
- <img src="https://example.com/image.png" />
+ <img src="https://example.com/image.png">
+ <img src="https://example.com/image.gif">
<script src="https://example.com/script.js" defer="defer">></script>
</body></html>`, { waitUntil: 'networkidle0' });
} finally {
@@ -192,6 +194,7 @@ describe('Apify.utils.puppeteer', () => {
expect(loadedUrls).to.have.members([
'https://example.com/image.png',
'https://example.com/script.js',
+ 'https://example.com/image.gif',
]);
});