Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Cosmetic code improvements, better logging, bumped package.json version
Jan Curnd9aa2e587ea8
diff --git a/package.json b/package.jsonindex 467602311..c0fd25dd1 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apifier",- "version": "0.0.49",+ "version": "0.0.50","description": "Apifier runtime for JavaScript","main": "build/index.js","keywords": [diff --git a/src/browser.js b/src/browser.jsindex 2f74b10ee..c439fec5b 100644--- a/src/browser.js+++ b/src/browser.js@@ -132,7 +132,7 @@ export class Browser {}).then((effectiveParsedProxyUrl) => {if (/^chrome$/i.test(this.options.browserName)) {- // In Chrome Capabilities.setProxy() has no effect,+ // In Chrome, Capabilities.setProxy() has no effect,// so we setup the proxy manuallythis.chromeOptions.addArguments(`--proxy-server=${effectiveParsedProxyUrl.host}`);} else {@@ -148,7 +148,7 @@ export class Browser {};this.capabilities.setProxy(proxyConfig);- console.dir(this.capabilities);+ // console.dir(this.capabilities);}});}@@ -166,7 +166,6 @@ export class Browser {}}).then(() => {- this.parsedChildProxyUrl = null;this.webDriver = null;});}diff --git a/src/proxy_chain.js b/src/proxy_chain.jsindex 8d8783ac3..63e1e908c 100644--- a/src/proxy_chain.js+++ b/src/proxy_chain.js@@ -21,7 +21,6 @@ export const PROXY_CHAIN = {TMP_DIR_TEMPLATE: '/tmp/apify-squid-XXXXXX',CONF_FILE_NAME: 'squid.conf',PID_FILE_NAME: 'squid.pid',- LOG_PREFIX: 'ProxyChain: ',};const tmpDirPromised = Promise.promisify(tmp.dir);@@ -116,7 +115,7 @@ export class ProxyChain {const cmd = PROXY_CHAIN.SQUID_CMD;const args = ['-f', squidConfPath, '-N'];const options = { cwd: this.tmpDir };- console.log(`${PROXY_CHAIN.LOG_PREFIX}Starting local proxy using: ${cmd} ${args.join(' ')}`);+ console.log(`Starting proxy chain: ${cmd} ${args.join(' ')}`);const proc = childProcess.spawn(cmd, args, options);this.squidPid = proc.pid;@@ -135,7 +134,7 @@ export class ProxyChain {proc.on('exit', (code, signal) => {if (isFinished) {- console.log(`${PROXY_CHAIN.LOG_PREFIX}Squid process ${this.squidPid} exited (code: ${code}, signal: ${signal})`);+ console.log(`Squid process ${this.squidPid} exited (code: ${code}, signal: ${signal})`);return;}@@ -147,7 +146,7 @@ export class ProxyChain {proc.on('error', (err) => {if (isFinished) {- console.log(`${PROXY_CHAIN.LOG_PREFIX}Squid process ${this.squidPid} failed: ${err}`);+ console.log(`Squid process ${this.squidPid} failed: ${err}`);return;}@@ -163,7 +162,7 @@ export class ProxyChain {// Print stdout/stderr to simplify debuggingconst printLog = (data) => {- console.log(`${PROXY_CHAIN.LOG_PREFIX}: Squid says: ${data}`);+ console.log(`Squid process ${this.squidPid} says: ${data}`);};proc.stdout.on('data', printLog);proc.stderr.on('data', printLog);@@ -171,7 +170,7 @@ export class ProxyChain {}).then(() => {const childProxyUrl = this._getChildProxyUrl();- console.log(`${PROXY_CHAIN.LOG_PREFIX}Started proxy chain ${childProxyUrl} => ${this._getParentProxyUrl()} (pid ${this.squidPid})`); // eslint-disable-line max-len+ console.log(`Started proxy chain ${childProxyUrl} => ${this._getParentProxyUrl()} (PID: ${this.squidPid}, tmpDir: ${this.tmpDir})`); // eslint-disable-line max-lenreturn parseUrl(childProxyUrl);});@@ -251,13 +250,13 @@ cache_peer_access my_peer allow all* @return {*}*/shutdown() {- console.log(`${PROXY_CHAIN.LOG_PREFIX}Shutting down proxy chain ${this._getChildProxyUrl()} => ${this._getParentProxyUrl()} (pid ${this.squidPid})`); // eslint-disable-line max-len+ console.log(`Shutting down proxy chain ${this._getChildProxyUrl()} => ${this._getParentProxyUrl()} (pid ${this.squidPid})`); // eslint-disable-line max-lenif (this._isSquidRunning()) {try {process.kill(this.squidPid, 'SIGKILL');} catch (err) {- console.log(`${PROXY_CHAIN.LOG_PREFIX}WARNING: Failed to kill Squid process ${this.squidPid}: ${err}`);+ console.log(`WARNING: Failed to kill Squid process ${this.squidPid}: ${err}`);}}@@ -265,7 +264,7 @@ cache_peer_access my_peer allow allconst tmpDir = this.tmpDir;rimraf(tmpDir, { glob: false }, (err) => {if (err) {- console.log(`${PROXY_CHAIN.LOG_PREFIX}WARNING: Failed to delete temporary directory at ${tmpDir}: ${err}`);+ console.log(`WARNING: Failed to delete temporary directory at ${tmpDir}: ${err}`);}});this.tmpDir = null;diff --git a/test/browser.js b/test/browser.jsindex aac391cc5..adcf9d902 100644--- a/test/browser.js+++ b/test/browser.js@@ -35,7 +35,7 @@ before(() => {}const parsed = basicAuthParser(auth);const isEqual = _.isEqual(parsed, proxyAuth);- console.log('parsed "Proxy-Authorization": %j - %s', parsed, isEqual);+ // console.log('parsed "Proxy-Authorization": %j - %s', parsed, isEqual);if (isEqual) wasProxyCalled = true;fn(null, isEqual);};