Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Commenting source
Marek Trunkat784cd04982f8
diff --git a/src/utils.js b/src/utils.jsindex e761c32e2..f8d4a673a 100644--- a/src/utils.js+++ b/src/utils.js@@ -61,6 +61,8 @@ export const requestPromise = (options) => {if (!request[method]) throw new Error('"options.method" is not a valid http request method');return new PromisesDependency((resolve, reject) => {+ // We have to use request[method]({ ... }) instead of request({ method, ... })+ // to be able to mock request when unit testing requestPromise().request[method](options, (error, response, body) => {if (error) return reject(error);@@ -70,7 +72,7 @@ export const requestPromise = (options) => {};/**- * Zips to arrays ['a1', 'a2', ... ] and ['b1', 'b2', ...] into map { a1: 'b1', a2: 'b2', ... }.+ * Zips two arrays ['a1', 'a2', ... ] and ['b1', 'b2', ...] into map { a1: 'b1', a2: 'b2', ... }.*/export const arrays2object = (keys, vals) => {return keys.reduce((prev, val, index) => {