Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/sdk improvements (#33)
Marek Trunkátcc97e2d10f4d
diff --git a/src/actor.js b/src/actor.jsindex 24a992014..98aa34825 100644--- a/src/actor.js+++ b/src/actor.js@@ -4,6 +4,7 @@ import fs from 'fs';import { checkParamOrThrow } from 'apify-client/build/utils';import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';import { newPromise, apifyClient, addCharsetToContentType } from './utils';+import { maybeStringify } from './key_value_store';/* globals process */@@ -296,26 +297,13 @@ export const call = (actId, input, opts = {}) => {checkParamOrThrow(build, 'build', 'Maybe String');if (build) runActOpts.build = build;- let { contentType } = opts;if (input) {- // TODO: this is duplicate with setValue()'s code- if (contentType === null || contentType === undefined) {- contentType = 'application/json';- try {- // Format JSON to simplify debugging, the overheads with compression is negligible- input = JSON.stringify(input, null, 2);- } catch (err) {- throw new Error(`The "input" parameter cannot be stringified to JSON: ${err.message}`);- }- if (input === undefined) {- throw new Error('The "input" parameter cannot be stringified to JSON.');- }- }+ input = maybeStringify(input, opts);checkParamOrThrow(input, 'input', 'Buffer|String');- checkParamOrThrow(contentType, 'contentType', 'String');+ checkParamOrThrow(opts.contentType, 'contentType', 'String');- if (contentType) runActOpts.contentType = addCharsetToContentType(contentType);+ if (opts.contentType) runActOpts.contentType = addCharsetToContentType(opts.contentType);runActOpts.body = input;}diff --git a/src/key_value_store.js b/src/key_value_store.jsindex 3b49b66e5..b6f331cbf 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -62,7 +62,7 @@ const validateSetValueParams = (key, value, options) => {** @ignore*/-const maybeStringify = (value, options) => {+export const maybeStringify = (value, options) => {// If contentType is missing, value will be stringified to JSONif (options.contentType === null || options.contentType === undefined) {options.contentType = 'application/json';diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex c066f0cee..cc6837c35 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -21,8 +21,6 @@ const DEFAULT_OPTIONS = {* Basic usage of PuppeteerCrawler:** ```javascript- * const request = require('request-promise');- ** const crawler = new Apify.PuppeteerCrawler({* requestList,* // Parameter page here is an intance of Puppeteer.Page with page.goto(request.url) already calleddiff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex f08fb9d53..bd1c06f8a 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -267,7 +267,7 @@ export default class PuppeteerPool {.then(browser => browser.newPage()).then((page) => {page.on('error', (error) => {- log.exception(error, 'PuppeteerPool: page crashled');+ log.exception(error, 'PuppeteerPool: page crashed');page.close();});@@ -278,6 +278,8 @@ export default class PuppeteerPool {.catch((err) => {log.exception(err, 'PuppeteerPool: browser.newPage() failed', { id: instance.id });this._retireInstance(instance);++ // !TODO: don't throw an error but repeat newPage with some delaythrow err;});}diff --git a/src/request_list.js b/src/request_list.jsindex 383180c18..262fb75a6 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -41,7 +41,7 @@ const ensureUniqueKeyValid = (uniqueKey) => {* Basic usage of RequestList:** ```javascript- * const requestList = new RequestList({+ * const requestList = new Apify.RequestList({* sources: [* // Seperate requests* { url: 'http://www.example.com/page-1', method: 'GET', headers: {} },@@ -260,15 +260,21 @@ export default class RequestList {return requestPromise.get(requestsFromUrl).then((urlsStr) => {const urlsArr = urlsStr.match(new RegExp(regex, 'g'));+ const originalLength = this.requests.length;++ urlsArr.forEach(url => this._addRequest(_.extend({ url }, sharedOpts)));++ const fetchedCount = urlsArr.length;+ const importedCount = this.requests.length - originalLength;log.info('RequestList: list fetched', {requestsFromUrl,regex,- count: urlsArr.length,+ fetchedCount,+ importedCount,+ dupliciteCount: fetchedCount - importedCount,sample: JSON.stringify(urlsArr.slice(0, 5)),});-- urlsArr.forEach(url => this._addRequest(_.extend({ url }, sharedOpts)));}).catch((err) => {log.exception(err, 'RequestList: Cannot fetch a request list', { requestsFromUrl, regex });diff --git a/test/key_value_store.js b/test/key_value_store.jsindex 5e3b03692..0de475935 100644--- a/test/key_value_store.js+++ b/test/key_value_store.js@@ -3,7 +3,7 @@ import chaiAsPromised from 'chai-as-promised';import 'babel-polyfill';import sinon from 'sinon';import { ENV_VARS } from '../build/constants';-import { KeyValueStoreLocal, KeyValueStore, LOCAL_EMULATION_SUBDIR } from '../build/key_value_store';+import { KeyValueStoreLocal, KeyValueStore, maybeStringify, LOCAL_EMULATION_SUBDIR } from '../build/key_value_store';import { apifyClient } from '../build/utils';import * as Apify from '../build/index';import { LOCAL_EMULATION_DIR, emptyLocalEmulationSubdir, expectNotLocalEmulation } from './_helper';@@ -16,6 +16,21 @@ describe('key_value_store', () => {beforeEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));afterEach(() => emptyLocalEmulationSubdir(LOCAL_EMULATION_SUBDIR));+ describe('maybeStringify()', () => {+ it('should work', () => {+ expect(maybeStringify({ foo: 'bar' }, { contentType: null })).to.be.eql('{\n "foo": "bar"\n}');+ expect(maybeStringify({ foo: 'bar' }, { contentType: undefined })).to.be.eql('{\n "foo": "bar"\n}');++ expect(maybeStringify('xxx', { contentType: undefined })).to.be.eql('"xxx"');+ expect(maybeStringify('xxx', { contentType: 'something' })).to.be.eql('xxx');++ const obj = {};+ obj.self = obj;+ expect(() => maybeStringify(obj, { contentType: null }))+ .to.throw('The "value" parameter cannot be stringified to JSON: Converting circular structure to JSON');+ });+ });+describe('local', async () => {it('should work', async () => {const store = new KeyValueStoreLocal('my-store-id', LOCAL_EMULATION_DIR);