Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Cosmetic code improvements, better logging, bumped package.json version

Jan Curnd9aa2e587ea8
diff --git a/package.json b/package.json
index 467602311..c0fd25dd1 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apifier",
- "version": "0.0.49",
+ "version": "0.0.50",
"description": "Apifier runtime for JavaScript",
"main": "build/index.js",
"keywords": [
diff --git a/src/browser.js b/src/browser.js
index 2f74b10ee..c439fec5b 100644
--- a/src/browser.js
+++ b/src/browser.js
@@ -132,7 +132,7 @@ export class Browser {
})
.then((effectiveParsedProxyUrl) => {
if (/^chrome$/i.test(this.options.browserName)) {
- // In Chrome Capabilities.setProxy() has no effect,
+ // In Chrome, Capabilities.setProxy() has no effect,
// so we setup the proxy manually
this.chromeOptions.addArguments(`--proxy-server=${effectiveParsedProxyUrl.host}`);
} else {
@@ -148,7 +148,7 @@ export class Browser {
};
this.capabilities.setProxy(proxyConfig);
- console.dir(this.capabilities);
+ // console.dir(this.capabilities);
}
});
}
@@ -166,7 +166,6 @@ export class Browser {
}
})
.then(() => {
- this.parsedChildProxyUrl = null;
this.webDriver = null;
});
}
diff --git a/src/proxy_chain.js b/src/proxy_chain.js
index 8d8783ac3..63e1e908c 100644
--- a/src/proxy_chain.js
+++ b/src/proxy_chain.js
@@ -21,7 +21,6 @@ export const PROXY_CHAIN = {
TMP_DIR_TEMPLATE: '/tmp/apify-squid-XXXXXX',
CONF_FILE_NAME: 'squid.conf',
PID_FILE_NAME: 'squid.pid',
- LOG_PREFIX: 'ProxyChain: ',
};
const tmpDirPromised = Promise.promisify(tmp.dir);
@@ -116,7 +115,7 @@ export class ProxyChain {
const cmd = PROXY_CHAIN.SQUID_CMD;
const args = ['-f', squidConfPath, '-N'];
const options = { cwd: this.tmpDir };
- console.log(`${PROXY_CHAIN.LOG_PREFIX}Starting local proxy using: ${cmd} ${args.join(' ')}`);
+ console.log(`Starting proxy chain: ${cmd} ${args.join(' ')}`);
const proc = childProcess.spawn(cmd, args, options);
this.squidPid = proc.pid;
@@ -135,7 +134,7 @@ export class ProxyChain {
proc.on('exit', (code, signal) => {
if (isFinished) {
- console.log(`${PROXY_CHAIN.LOG_PREFIX}Squid process ${this.squidPid} exited (code: ${code}, signal: ${signal})`);
+ console.log(`Squid process ${this.squidPid} exited (code: ${code}, signal: ${signal})`);
return;
}
@@ -147,7 +146,7 @@ export class ProxyChain {
proc.on('error', (err) => {
if (isFinished) {
- console.log(`${PROXY_CHAIN.LOG_PREFIX}Squid process ${this.squidPid} failed: ${err}`);
+ console.log(`Squid process ${this.squidPid} failed: ${err}`);
return;
}
@@ -163,7 +162,7 @@ export class ProxyChain {
// Print stdout/stderr to simplify debugging
const printLog = (data) => {
- console.log(`${PROXY_CHAIN.LOG_PREFIX}: Squid says: ${data}`);
+ console.log(`Squid process ${this.squidPid} says: ${data}`);
};
proc.stdout.on('data', printLog);
proc.stderr.on('data', printLog);
@@ -171,7 +170,7 @@ export class ProxyChain {
})
.then(() => {
const childProxyUrl = this._getChildProxyUrl();
- console.log(`${PROXY_CHAIN.LOG_PREFIX}Started proxy chain ${childProxyUrl} => ${this._getParentProxyUrl()} (pid ${this.squidPid})`); // eslint-disable-line max-len
+ console.log(`Started proxy chain ${childProxyUrl} => ${this._getParentProxyUrl()} (PID: ${this.squidPid}, tmpDir: ${this.tmpDir})`); // eslint-disable-line max-len
return parseUrl(childProxyUrl);
});
@@ -251,13 +250,13 @@ cache_peer_access my_peer allow all
* @return {*}
*/
shutdown() {
- console.log(`${PROXY_CHAIN.LOG_PREFIX}Shutting down proxy chain ${this._getChildProxyUrl()} => ${this._getParentProxyUrl()} (pid ${this.squidPid})`); // eslint-disable-line max-len
+ console.log(`Shutting down proxy chain ${this._getChildProxyUrl()} => ${this._getParentProxyUrl()} (pid ${this.squidPid})`); // eslint-disable-line max-len
if (this._isSquidRunning()) {
try {
process.kill(this.squidPid, 'SIGKILL');
} catch (err) {
- console.log(`${PROXY_CHAIN.LOG_PREFIX}WARNING: Failed to kill Squid process ${this.squidPid}: ${err}`);
+ console.log(`WARNING: Failed to kill Squid process ${this.squidPid}: ${err}`);
}
}
@@ -265,7 +264,7 @@ cache_peer_access my_peer allow all
const tmpDir = this.tmpDir;
rimraf(tmpDir, { glob: false }, (err) => {
if (err) {
- console.log(`${PROXY_CHAIN.LOG_PREFIX}WARNING: Failed to delete temporary directory at ${tmpDir}: ${err}`);
+ console.log(`WARNING: Failed to delete temporary directory at ${tmpDir}: ${err}`);
}
});
this.tmpDir = null;
diff --git a/test/browser.js b/test/browser.js
index aac391cc5..adcf9d902 100644
--- a/test/browser.js
+++ b/test/browser.js
@@ -35,7 +35,7 @@ before(() => {
}
const parsed = basicAuthParser(auth);
const isEqual = _.isEqual(parsed, proxyAuth);
- console.log('parsed "Proxy-Authorization": %j - %s', parsed, isEqual);
+ // console.log('parsed "Proxy-Authorization": %j - %s', parsed, isEqual);
if (isEqual) wasProxyCalled = true;
fn(null, isEqual);
};