Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Updated utils.newClient() to only use the new apify-client 'baseUrl' option, updated unit test
Jan Curne1ef48165828
diff --git a/src/utils.js b/src/utils.jsindex 8f77e99b6..c9f08c390 100644--- a/src/utils.js+++ b/src/utils.js@@ -69,19 +69,11 @@ export const nodeifyPromise = (promise, callback) => {* @return {*}*/export const newClient = () => {- // TODO: protocol/host/port/basePath should be replaced with baseUrl- const opts = {};- if (process.env[APIFY_ENV_VARS.API_BASE_URL]) {- const parsed = url.parse(process.env[APIFY_ENV_VARS.API_BASE_URL]);- opts.protocol = parsed.protocol.replace(':', '');- opts.host = parsed.hostname;- opts.port = parsed.port ? parseInt(parsed.port, 10) : null;- opts.basePath = parsed.pathname.replace(/\/$/, '');- opts.baseUrl = process.env[APIFY_ENV_VARS.API_BASE_URL];- }-- opts.userId = process.env[APIFY_ENV_VARS.USER_ID] || null;- opts.token = process.env[APIFY_ENV_VARS.TOKEN] || null;+ const opts = {+ baseUrl: process.env[APIFY_ENV_VARS.API_BASE_URL] || null,+ userId: process.env[APIFY_ENV_VARS.USER_ID] || null,+ token: process.env[APIFY_ENV_VARS.TOKEN] || null,+ };return new ApifyClient(opts);};diff --git a/test/utils.js b/test/utils.jsindex 957eb65cc..7203b34a7 100644--- a/test/utils.js+++ b/test/utils.js@@ -36,8 +36,7 @@ describe('Apifier.xxxPromisesDependency()', () => {describe('utils.newClient', () => {it('reads environment variables correctly', () => {- // TODO: after new version of client is deployed, add '/' at the end of this path, client should remove it!- process.env.APIFY_API_BASE_URL = 'http://www.example.com:1234/path';+ process.env.APIFY_API_BASE_URL = 'http://www.example.com:1234/path/';process.env.APIFY_USER_ID = 'userId';process.env.APIFY_TOKEN = 'token';const client = utils.newClient();@@ -47,6 +46,6 @@ describe('utils.newClient', () => {expect(opts.userId).to.eql('userId');expect(opts.token).to.eql('token');- expect(opts.baseUrl).to.eql('http://www.example.com:1234/path');+ expect(opts.baseUrl).to.eql('http://www.example.com:1234/path/');});});