Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Switched to new "apify-client" version, bumped package.json version, development of Apifier.browse() proxy settings
Jan Curncc1f0bf6db7f
diff --git a/README.md b/README.mdindex 1eb60a76e..1370d9248 100644--- a/README.md+++ b/README.md@@ -122,7 +122,7 @@ argument called `context` which is an object such as:// Input data for the act as provided by Apifier.getInput()input: {- body: String/Buffer,+ body: Object,contentType: String,}}@@ -142,10 +142,10 @@ Apifier.getContext().then((context) => {### Input and output-Each act can have an input and output data record, which is a string or binary buffer+Each act can have an input and output data record, which is a raw datawith a specific MIME content type.Both input and output is stored in the Apifier key-value store created specifically for the act run,-under keys `INPUT` and `OUTPUT`, respectively.+under keys named `INPUT` and `OUTPUT`, respectively.The ID of the key-value store is provided by the Actor runtime as the `APIFY_DEFAULT_KEY_VALUE_STORE_ID`environment variable.@@ -158,6 +158,9 @@ Apifier.getInput().then((input) => {});```+If the input data has the `application/json' content type, it is automatically parsed into JavaScript object,+otherwise it is left as a String or Buffer in the raw form.+Similarly, the output can be stored as follows:```javascript@@ -192,7 +195,7 @@ It has the following properties:{// The type of the web browser to use.// See https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities for possible options.- // By default it is 'chrome'.+ // By default it is 'chrome', which is currently the only fully-supported browser.browserName: String,// Indicates whether the browser should be opened in headless mode (i.e. without windows).diff --git a/package.json b/package.jsonindex a52148bde..cfd998f8a 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apifier",- "version": "0.0.44",+ "version": "0.0.45","description": "Apifier runtime for JavaScript","main": "build/index.js","keywords": [@@ -40,7 +40,8 @@"lint": "npm run build && eslint src test"},"dependencies": {- "apify-client": ">=0.1.6"+ "apify-client": ">=0.1.6",+ "underscore": "^1.8.3"},"devDependencies": {"babel-cli": "^6.6.5",diff --git a/src/actor.js b/src/actor.jsindex 0c18485b8..64bf4e2df 100644--- a/src/actor.js+++ b/src/actor.js@@ -1,9 +1,17 @@import fs from 'fs';+import _ from 'underscore';import { APIFY_ENV_VARS, EXIT_CODES, KV_STORE_KEYS } from './constants';import { getPromisePrototype, newPromise, nodeifyPromise, newClient } from './utils';/* global process, Buffer */++const JSON_CONTENT_TYPES = [+ 'application/json',+ 'application/json; charset=utf-8',+];++/*** A default instance of ApifyClient class.* It can be configured by calling setOptions() function.@@ -47,14 +55,21 @@ export const getInput = (callback = null) => {});}).then((input) => {- // Ensure we always return null or { body: String|Buffer, contentType: String|null } to user+ // Ensure we always return either:+ // * null+ // * or { body: String|Buffer, contentType: String|null }+ // * or { body: Any, contentType: 'application/json' }+ const baseMsg = 'ApifyClient returned an unexpected value from keyValueStores.getRecord()';if (!input) {input = null;- } else if (typeof (input) !== 'object'- || (typeof (input.body) !== 'string' && !Buffer.isBuffer(input.body))- || (typeof (input.contentType) !== 'string' && input.contentType !== null)) {- console.log(input);- throw new Error('ApifyClient returned an unexpected value from keyValueStores.getRecord()');+ } else if (typeof (input) !== 'object') {+ throw new Error(`${baseMsg}: expected an object.`);+ } else if ((typeof (input.contentType) !== 'string' && input.contentType !== null)) {+ throw new Error(`${baseMsg}: contentType is not valid.`);+ } else if (!_.contains(JSON_CONTENT_TYPES, input.contentType)+ && typeof (input.body) !== 'string'+ && !Buffer.isBuffer(input.body)) {+ throw new Error(`${baseMsg}: body must be String or Buffer.`);}return input;});diff --git a/src/browser.js b/src/browser.jsindex 0cc4a3134..8134676ae 100644--- a/src/browser.js+++ b/src/browser.js@@ -62,9 +62,11 @@ export const browse = (url, options = null, callback = null) => {options = Object.assign(getDefaultBrowseOptions(), options);// This is an optional dependency because it is quite large, only require it when used!- const { Builder } = require('selenium-webdriver'); // eslint-disable-line global-require+ const { Builder, Capabilities, logging } = require('selenium-webdriver'); // eslint-disable-line global-requireconst chrome = require('selenium-webdriver/chrome'); // eslint-disable-line global-require+ logging.installConsoleHandler();+// By default, Selenium already defines a long list of command-line options// to enable browser automation, here we add a few other ones// (inspired by Lighthouse, see lighthouse/lighthouse-cli/chrome-launcher)@@ -72,9 +74,8 @@ export const browse = (url, options = null, callback = null) => {// Define capabilities of the web browser,// see https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities for reference.- const capabilities = {- browserName: options.browserName,- };+ const caps = new Capabilities();+ caps.set('browserName', options.browserName);// Disable built-in Google Translate servicechromeOpts.addArguments('--disable-translate');@@ -100,24 +101,38 @@ export const browse = (url, options = null, callback = null) => {// NOTE: to view effective proxy settings in Chrome, open chrome://net-internals/#proxy// https://sites.google.com/a/chromium.org/chromedriver/capabilities-- capabilities.proxy = {- proxyType: 'manual',- httpProxy: parsed.host,- ftpProxy: parsed.host,- sslProxy: parsed.host,- socksProxy: parsed.host,- socksUsername: parsed.username,- socksPassword: parsed.password,- noProxy: '', // Do not skip proxy for any address- };-- console.dir(capabilities);+ // https://github.com/haad/proxychains/blob/0f61bd071389398a4c8378847a45973577593e6f/src/proxychains.conf+ // https://www.rootusers.com/configure-squid-proxy-to-forward-to-a-parent-proxy/+ // https://gist.github.com/leefsmp/3e4385e08ea27e30ba96+ // https://github.com/tinyproxy/tinyproxy++ // 1) install pkgsrc: http://pkgsrc.joyent.com/+ // 2) install tinyproxy: sudo pkgin -y install tinyproxy++ if (/^chrome$/i.test(options.browserName)) {+ // In Chrome Capabilities.setProxy() has no effect,+ // so we setup the proxy manually+ chromeOpts.addArguments(`--proxy-server=${parsed.host}`);+ } else {+ const proxyConfig = {+ proxyType: 'MANUAL',+ httpProxy: parsed.host,+ sslProxy: parsed.host,+ ftpProxy: parsed.host,+ // socksProxy: parsed.host,+ //socksUsername: parsed.username,+ //socksPassword: parsed.password,+ // noProxy: '', // Do not skip proxy for any address+ };+ caps.setProxy(proxyConfig);++ console.dir(caps);+ }}const webDriver = new Builder().setChromeOptions(chromeOpts)- .withCapabilities(capabilities)+ .withCapabilities(caps).build();const browser = new Browser(webDriver);diff --git a/test/browser.js b/test/browser.jsindex 5a0974f7c..28479e93c 100644--- a/test/browser.js+++ b/test/browser.js@@ -51,7 +51,8 @@ before(() => {});});-after(() => {+after(function () {+ this.timeout(60 * 1000);if (proxyServer) return Promise.promisify(proxyServer.close).bind(proxyServer)();});@@ -149,13 +150,13 @@ describe('Apifier.browse()', function () {});});- /* TODO !!!!+ /*it('works with proxy server', () => {let browser;wasProxyCalled = false;const opts = {headless: false,- browserName: 'firefox',+ browserName: 'chrome',proxyUrl: `http://${proxyAuth.username}:${proxyAuth.password}@127.0.0.1:${proxyPort}`,};return Apifier.browse('https://www.example.com', opts)@@ -163,11 +164,14 @@ describe('Apifier.browse()', function () {browser = res;}).then(() => {- return new Promise((resolve) => {- setTimeout(resolve, 60 * 1000);- });+ return browser.webDriver.sleep(60 * 1000);}).then(() => {+ return browser.webDriver.getAllWindowHandles();+ })+ .then((handles) => {+ console.dir(handles);+expect(browser.constructor.name).to.eql('Browser');return browser.webDriver.getCurrentUrl();})@@ -176,8 +180,7 @@ describe('Apifier.browse()', function () {expect(url).to.eql('https://www.example.com/');return browser.close();});- });- */+ }); */});