Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Updated utils.newClient() to only use the new apify-client 'baseUrl' option, updated unit test

Jan Curne1ef48165828
diff --git a/src/utils.js b/src/utils.js
index 8f77e99b6..c9f08c390 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -69,19 +69,11 @@ export const nodeifyPromise = (promise, callback) => {
* @return {*}
*/
export const newClient = () => {
- // TODO: protocol/host/port/basePath should be replaced with baseUrl
- const opts = {};
- if (process.env[APIFY_ENV_VARS.API_BASE_URL]) {
- const parsed = url.parse(process.env[APIFY_ENV_VARS.API_BASE_URL]);
- opts.protocol = parsed.protocol.replace(':', '');
- opts.host = parsed.hostname;
- opts.port = parsed.port ? parseInt(parsed.port, 10) : null;
- opts.basePath = parsed.pathname.replace(/\/$/, '');
- opts.baseUrl = process.env[APIFY_ENV_VARS.API_BASE_URL];
- }
-
- opts.userId = process.env[APIFY_ENV_VARS.USER_ID] || null;
- opts.token = process.env[APIFY_ENV_VARS.TOKEN] || null;
+ const opts = {
+ baseUrl: process.env[APIFY_ENV_VARS.API_BASE_URL] || null,
+ userId: process.env[APIFY_ENV_VARS.USER_ID] || null,
+ token: process.env[APIFY_ENV_VARS.TOKEN] || null,
+ };
return new ApifyClient(opts);
};
diff --git a/test/utils.js b/test/utils.js
index 957eb65cc..7203b34a7 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -36,8 +36,7 @@ describe('Apifier.xxxPromisesDependency()', () => {
describe('utils.newClient', () => {
it('reads environment variables correctly', () => {
- // TODO: after new version of client is deployed, add '/' at the end of this path, client should remove it!
- process.env.APIFY_API_BASE_URL = 'http://www.example.com:1234/path';
+ process.env.APIFY_API_BASE_URL = 'http://www.example.com:1234/path/';
process.env.APIFY_USER_ID = 'userId';
process.env.APIFY_TOKEN = 'token';
const client = utils.newClient();
@@ -47,6 +46,6 @@ describe('utils.newClient', () => {
expect(opts.userId).to.eql('userId');
expect(opts.token).to.eql('token');
- expect(opts.baseUrl).to.eql('http://www.example.com:1234/path');
+ expect(opts.baseUrl).to.eql('http://www.example.com:1234/path/');
});
});