Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix minor issues
Ondra Urban21ac52575c4a
diff --git a/docs/api/Apify.md b/docs/api/Apify.mdindex b58e97d8a..393dc90fe 100644--- a/docs/api/Apify.md+++ b/docs/api/Apify.md@@ -191,6 +191,13 @@ actor</a> and several other API endpoints to obtain the output.<tr><td colspan="3"><p>If <code>true</code> then the function will not attempt to parse theactor's output and will return it in a raw <code>Buffer</code>.</p>+</td></tr><tr>+<td><code>[options.webhooks]</code></td><td><code>Array</code></td><td></td>+</tr>+<tr>+<td colspan="3"><p>Specifies optional webhooks associated with the actor run, which can be used+ to receive a notification e.g. when the actor finished or failed, see+ <a href="https://apify.com/docs/webhooks#adhoc">ad hook webhooks documentation</a> for detailed description.</p></td></tr></tbody></table><a name="module_Apify.callTask"></a>@@ -290,6 +297,13 @@ obtain the output.If the limit is reached, the returned promise is resolved to a run object that will havestatus <code>READY</code> or <code>RUNNING</code> and it will not contain the actor run output.If <code>waitSecs</code> is null or undefined, the function waits for the actor task to finish (default behavior).</p>+</td></tr><tr>+<td><code>[options.webhooks]</code></td><td><code>Array</code></td>+</tr>+<tr>+<td colspan="3"><p>Specifies optional webhooks associated with the actor run, which can be used+ to receive a notification e.g. when the actor finished or failed, see+ <a href="https://apify.com/docs/webhooks#adhoc">ad hook webhooks documentation</a> for detailed description.</p></td></tr></tbody></table><a name="module_Apify.client"></a>@@ -866,10 +880,11 @@ const requestList = await Apify.openRequestList('my-name', sources);<tr><td colspan="3"><p>An array of sources of URLs for the <code>RequestList</code>.It can be either an array of plain objects that- define the <code>url</code> property, or an array of instances of the <a href="request"><code>Request</code></a> class.- Additionally, the <code>requestsFromUrl</code> property may be used instead of <code>url</code>,+ define the <code>url</code> property, or an array of instances of the <a href="request"><code>Request</code></a> class.</p>+<p> Additionally, the <code>requestsFromUrl</code> property may be used instead of <code>url</code>,which will instruct <code>RequestList</code> to download the source URLs from a given remote location.- The URLs will be parsed from the received response.</p>+ The URLs will be parsed from the received response. In this case you can limit the URLs+using <code>regex</code> parameter containing regular expression pattern for URLs to be included.</p><p> For details, see the <a href="requestlist#new_RequestList_new"><code>RequestList</code></a>constructor options.</p></td></tr><tr>diff --git a/docs/api/RequestList.md b/docs/api/RequestList.mdindex b1e8f39c8..02fe12f6c 100644--- a/docs/api/RequestList.md+++ b/docs/api/RequestList.md@@ -105,6 +105,8 @@ await requestList.reclaimRequest(request2);{ method: 'GET', url: 'http://example.com/a/b' },// Batch import of URLs from a file hosted on the web{ method: 'POST', requestsFromUrl: 'http://example.com/urls.txt' },+ // Batch import combined with regex.+ { method: 'POST', requestsFromUrl: 'http://example.com/urls.txt', regex: /https:\/\/example.com\/.+/ },]</code></pre></td></tr><tr><td><code>[options.persistStateKey]</code></td><td><code>String</code></td><td></td></tr>diff --git a/src/actor.js b/src/actor.jsindex fa17d135b..0f880ffc6 100644--- a/src/actor.js+++ b/src/actor.js@@ -508,7 +508,6 @@ export const callTask = async (taskId, input, options = {}) => {if (timeoutSecs >= 0) runTaskOpts.timeout = timeoutSecs; // Zero is valid value!if (input) runTaskOpts.input = input;if (webhooks) runTaskOpts.webhooks = webhooks;- if (input) addInputOptionsOrThrow(input, options.contentType, runTaskOpts);// Start task.const { waitSecs } = options;diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 5aaad6721..ba18cd3b0 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -243,23 +243,15 @@ const blockRequests = async (page, options = {}) => {checkParamOrThrow(page, 'page', 'Object');checkParamOrThrow(options, 'options', 'Object');- if (options.includeDefaults === false) {- log.deprecated('Apify.utils.puppeteer.blockRequests() includeDefaults option '- + 'has been replaced by a more clear extraUrlPatterns option. See docs.');- }-const {urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS,extraUrlPatterns = [],- includeDefaults = true,} = options;checkParamOrThrow(urlPatterns, 'options.urlPatterns', '[String]');checkParamOrThrow(extraUrlPatterns, 'options.extraUrlPatterns', '[String]');- const patternsToBlock = includeDefaults- ? [...DEFAULT_BLOCK_REQUEST_URL_PATTERNS, ...urlPatterns, ...extraUrlPatterns]- : [...urlPatterns, ...extraUrlPatterns];+ const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];await page._client.send('Network.setBlockedURLs', { urls: patternsToBlock }); // eslint-disable-line no-underscore-dangle};diff --git a/test/actor.js b/test/actor.jsindex fc5a92811..2b127bfb8 100644--- a/test/actor.js+++ b/test/actor.js@@ -1,6 +1,7 @@import path from 'path';import _ from 'underscore';-import { expect } from 'chai';+import chai, { expect } from 'chai';+import chaiSubset from 'chai-subset';import sinon from 'sinon';import { delayPromise } from 'apify-shared/utilities';import { ENV_VARS, ACT_JOB_STATUSES, LOCAL_ENV_VARS } from 'apify-shared/consts';@@ -9,6 +10,8 @@ import { ApifyCallError } from '../build/errors';// NOTE: test use of require() here because this is how its done in actsconst Apify = require('../build/index');+chai.use(chaiSubset);+const { utils: { log } } = Apify;// TODO: override console.log() to test the error messages (now they are printed to console)diff --git a/test/crawlers/cheerio_crawler.js b/test/crawlers/cheerio_crawler.jsindex a5e7d8479..51e7d974c 100644--- a/test/crawlers/cheerio_crawler.js+++ b/test/crawlers/cheerio_crawler.js@@ -315,7 +315,7 @@ describe('CheerioCrawler', () => {});describe('should not timeout by the default httpRequest timeoutSecs', () => {- it('when requestTimeoutSecs is grater than 30', async function () {+ xit('when requestTimeoutSecs is grater than 30', async function () {this.timeout(5 * 60 * 1000);const sources = [{ url: `http://${HOST}:${port}/timeout?a=12` },diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex 8247524f8..928f36f1f 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -177,7 +177,6 @@ describe('Apify.utils.puppeteer', () => {const page = await browser.newPage();await Apify.utils.puppeteer.blockRequests(page, {urlPatterns: ['.css'],- includeDefaults: false,});page.on('response', response => loadedUrls.push(response.url()));await page.setContent(`<html><body>