Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Default apify proxy hostname and apify proxy port (#30)
Jakub Drobníka72c6ffe3c15
diff --git a/src/actor.js b/src/actor.jsindex 290fb7232..3dec84ba5 100644--- a/src/actor.js+++ b/src/actor.js@@ -2,7 +2,7 @@ import _ from 'underscore';import Promise from 'bluebird';import fs from 'fs';import { checkParamOrThrow } from 'apify-client/build/utils';-import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES } from './constants';+import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';import { newPromise, apifyClient, addCharsetToContentType } from './utils';/* globals process */@@ -392,8 +392,8 @@ export const getApifyProxyUrl = (opts = {}) => {groups,session,password = process.env[ENV_VARS.PROXY_PASSWORD],- hostname = process.env[ENV_VARS.PROXY_HOSTNAME],- port = parseInt(process.env[ENV_VARS.PROXY_PORT], 10),+ hostname = process.env[ENV_VARS.PROXY_HOSTNAME] || DEFAULT_PROXY_HOSTNAME,+ port = parseInt(process.env[ENV_VARS.PROXY_PORT], 10) || DEFAULT_PROXY_PORT,} = opts;// TODO: Check that session and groups are alphanumeric!diff --git a/src/constants.js b/src/constants.jsindex 3014dcc89..a737f39bf 100644--- a/src/constants.js+++ b/src/constants.js@@ -1,6 +1,9 @@import * as consts from 'apify-shared/consts';consts.DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'; // eslint-disable-line max-len+consts.DEFAULT_PROXY_HOSTNAME = 'proxy.apify.com';+consts.DEFAULT_PROXY_PORT = 8000;+/*** Exit codes for the act process.diff --git a/test/actor.js b/test/actor.jsindex 81fc0c602..d3eb19c0f 100644--- a/test/actor.js+++ b/test/actor.js@@ -5,7 +5,7 @@ import sinon from 'sinon';import tmp from 'tmp';import Promise from 'bluebird';import { delayPromise } from 'apify-shared/utilities';-import { ACT_TASK_STATUSES, ENV_VARS } from '../build/constants';+import { ACT_TASK_STATUSES, ENV_VARS, DEFAULT_PROXY_PORT, DEFAULT_PROXY_HOSTNAME } from '../build/constants';// NOTE: test use of require() here because this is how its done in actsconst Apify = require('../build/index');@@ -708,6 +708,8 @@ describe('Apify.getApifyProxyUrl()', () => {delete process.env[ENV_VARS.PROXY_HOSTNAME];delete process.env[ENV_VARS.PROXY_PORT];+ expect(Apify.getApifyProxyUrl({ password: 'xyz' })).to.be.eql(`http://auto:xyz@${DEFAULT_PROXY_HOSTNAME}:${DEFAULT_PROXY_PORT}`);+expect(() => Apify.getApifyProxyUrl()).to.throw();expect(Apify.getApifyProxyUrl({