Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: proxyUrl without port throwing on browser launch (#1065)

Ondra Urbanea100959dcb9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6949fe264..25c9c95d2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -41,6 +41,12 @@ const dataset = await Apify.openDataset('custom-name');
await dataset.pushData({ myValue: 123 });
```
+## Full list of changes:
+- Add `Configuration` class and `Apify` named export, see above.
+- Fix `proxyUrl` without a port throwing an error when launching browsers.
+- Fix `maxUsageCount` of a `Session` not being persisted.
+- Update `puppeteer` and `playwright` to match stable Chrome (90).
+
1.2.1 / 2021/05/14
====================
- Fix `requestAsBrowser` behavior with various combinations of `json`, `payload` legacy options.
diff --git a/src/browser_launchers/browser_launcher.js b/src/browser_launchers/browser_launcher.js
index 31468f319..01910157e 100644
--- a/src/browser_launchers/browser_launcher.js
+++ b/src/browser_launchers/browser_launcher.js
@@ -73,7 +73,7 @@ export default class BrowserLauncher {
launchOptions = {},
} = launchContext;
- this._validateProxyUrl(proxyUrl);
+ this._validateProxyUrlProtocol(proxyUrl);
// those need to be reassigned otherwise they are {} in types
/** @type {*} */
@@ -154,17 +154,11 @@ export default class BrowserLauncher {
* @param {string} proxyUrl
* @private
*/
- _validateProxyUrl(proxyUrl) {
- if (!proxyUrl) {
- return;
- }
+ _validateProxyUrlProtocol(proxyUrl) {
+ if (!proxyUrl) return;
- const parsedProxyUrl = new URL(proxyUrl);
- if (!parsedProxyUrl.host || !parsedProxyUrl.port) {
- throw new Error('Invalid "proxyUrl" option: both hostname and port must be provided.');
- }
- if (!/^(http|https|socks4|socks5)$/.test(parsedProxyUrl.protocol.replace(':', ''))) {
- throw new Error(`Invalid "proxyUrl" option: Unsupported scheme (${parsedProxyUrl.protocol.replace(':', '')}).`);
+ if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {
+ throw new Error(`Invalid "proxyUrl". Unsupported protocol: ${proxyUrl}.`);
}
}
}
diff --git a/test/browser_launchers/playwright_launcher.test.js b/test/browser_launchers/playwright_launcher.test.js
index e3eda2e5e..6f2bbf5cd 100644
--- a/test/browser_launchers/playwright_launcher.test.js
+++ b/test/browser_launchers/playwright_launcher.test.js
@@ -62,33 +62,34 @@ afterAll(() => {
}, 5000);
describe('Apify.launchPlaywright()', () => {
- test('throws on invalid args', () => {
- expect(Apify.launchPlaywright('some non-object')).rejects.toThrow(Error);
- expect(Apify.launchPlaywright(1234)).rejects.toThrow(Error);
-
- expect(Apify.launchPlaywright({ proxyUrl: 234 })).rejects.toThrow(Error);
- expect(Apify.launchPlaywright({ proxyUrl: {} })).rejects.toThrow(Error);
- expect(Apify.launchPlaywright({ proxyUrl: 'invalidurl' })).rejects.toThrow(Error);
- expect(Apify.launchPlaywright({ proxyUrl: 'http://host-without-port' })).rejects.toThrow(Error);
- expect(Apify.launchPlaywright({ proxyUrl: 'invalid://somehost:1234' })).rejects.toThrow(Error);
- expect(Apify.launchPlaywright({ proxyUrl: 'https://user:[email protected]:1234' })).rejects.toThrow(Error);
- expect(Apify.launchPlaywright({ proxyUrl: 'socks4://user:[email protected]:1234' })).rejects.toThrow(Error);
- expect(Apify.launchPlaywright({ proxyUrl: 'socks5://user:[email protected]:1234' })).rejects.toThrow(Error);
- expect(Apify.launchPlaywright({ proxyUrl: ' something really bad' })).rejects.toThrow(Error);
+ test('throws on invalid args', async () => {
+ await expect(Apify.launchPlaywright('some non-object')).rejects.toThrow(Error);
+ await expect(Apify.launchPlaywright(1234)).rejects.toThrow(Error);
+
+ await expect(Apify.launchPlaywright({ proxyUrl: 234 })).rejects.toThrow(Error);
+ await expect(Apify.launchPlaywright({ proxyUrl: {} })).rejects.toThrow(Error);
+ await expect(Apify.launchPlaywright({ proxyUrl: 'invalidurl' })).rejects.toThrow(Error);
+ await expect(Apify.launchPlaywright({ proxyUrl: 'invalid://somehost:1234' })).rejects.toThrow(Error);
+ await expect(Apify.launchPlaywright({ proxyUrl: 'https://user:[email protected]:1234' })).rejects.toThrow(Error);
+ await expect(Apify.launchPlaywright({ proxyUrl: 'socks4://user:[email protected]:1234' })).rejects.toThrow(Error);
+ await expect(Apify.launchPlaywright({ proxyUrl: 'socks5://user:[email protected]:1234' })).rejects.toThrow(Error);
+ await expect(Apify.launchPlaywright({ proxyUrl: ' something really bad' })).rejects.toThrow(Error);
});
- test('opens supports non-HTTP proxies without authentication', async () => {
+ test('supports non-HTTP proxies without authentication', async () => {
+ const closePromises = [];
const browser1 = await Apify.launchPlaywright({ proxyUrl: 'socks4://example.com:1234' });
- browser1.close();
+ closePromises.push(browser1.close());
const browser2 = await Apify.launchPlaywright({ proxyUrl: 'socks5://example.com:1234' });
- browser2.close();
+ closePromises.push(browser2.close());
const browser3 = await Apify.launchPlaywright({ proxyUrl: 'https://example.com:1234' });
- browser3.close();
+ closePromises.push(browser3.close());
const browser4 = await Apify.launchPlaywright({ proxyUrl: 'HTTP://example.com:1234' });
- browser4.close();
+ closePromises.push(browser4.close());
+ await Promise.all(closePromises);
});
test('opens https://www.example.com', async () => {
diff --git a/test/browser_launchers/puppeteer_launcher.test.js b/test/browser_launchers/puppeteer_launcher.test.js
index ef24d6f85..db46da34e 100644
--- a/test/browser_launchers/puppeteer_launcher.test.js
+++ b/test/browser_launchers/puppeteer_launcher.test.js
@@ -60,36 +60,37 @@ afterAll(() => {
}, 5000);
describe('Apify.launchPuppeteer()', () => {
- test('throws on invalid args', () => {
- expect(Apify.launchPuppeteer('some non-object')).rejects.toThrow(Error);
- expect(Apify.launchPuppeteer(1234)).rejects.toThrow(Error);
-
- expect(Apify.launchPuppeteer({ proxyUrl: 234 })).rejects.toThrow(Error);
- expect(Apify.launchPuppeteer({ proxyUrl: {} })).rejects.toThrow(Error);
- expect(Apify.launchPuppeteer({ proxyUrl: 'invalidurl' })).rejects.toThrow(Error);
- expect(Apify.launchPuppeteer({ proxyUrl: 'http://host-without-port' })).rejects.toThrow(Error);
- expect(Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).rejects.toThrow(Error);
- expect(Apify.launchPuppeteer({ proxyUrl: 'https://user:[email protected]:1234' })).rejects.toThrow(Error);
- expect(Apify.launchPuppeteer({ proxyUrl: 'socks4://user:[email protected]:1234' })).rejects.toThrow(Error);
- expect(Apify.launchPuppeteer({ proxyUrl: 'socks5://user:[email protected]:1234' })).rejects.toThrow(Error);
- expect(Apify.launchPuppeteer({ proxyUrl: ' something really bad' })).rejects.toThrow(Error);
-
- expect(Apify.launchPuppeteer({ launchOptions: { args: 'wrong args' } })).rejects.toThrow(Error);
- expect(Apify.launchPuppeteer({ launchOptions: { args: [12, 34] } })).rejects.toThrow(Error);
+ test('throws on invalid args', async () => {
+ await expect(Apify.launchPuppeteer('some non-object')).rejects.toThrow(Error);
+ await expect(Apify.launchPuppeteer(1234)).rejects.toThrow(Error);
+
+ await expect(Apify.launchPuppeteer({ proxyUrl: 234 })).rejects.toThrow(Error);
+ await expect(Apify.launchPuppeteer({ proxyUrl: {} })).rejects.toThrow(Error);
+ await expect(Apify.launchPuppeteer({ proxyUrl: 'invalidurl' })).rejects.toThrow(Error);
+ await expect(Apify.launchPuppeteer({ proxyUrl: 'invalid://somehost:1234' })).rejects.toThrow(Error);
+ await expect(Apify.launchPuppeteer({ proxyUrl: 'https://user:[email protected]:1234' })).rejects.toThrow(Error);
+ await expect(Apify.launchPuppeteer({ proxyUrl: 'socks4://user:[email protected]:1234' })).rejects.toThrow(Error);
+ await expect(Apify.launchPuppeteer({ proxyUrl: 'socks5://user:[email protected]:1234' })).rejects.toThrow(Error);
+ await expect(Apify.launchPuppeteer({ proxyUrl: ' something really bad' })).rejects.toThrow(Error);
+
+ await expect(Apify.launchPuppeteer({ launchOptions: { args: 'wrong args' } })).rejects.toThrow(Error);
+ await expect(Apify.launchPuppeteer({ launchOptions: { args: [12, 34] } })).rejects.toThrow(Error);
});
- test('opens supports non-HTTP proxies without authentication', async () => {
+ test('supports non-HTTP proxies without authentication', async () => {
+ const closePromises = [];
const browser1 = await Apify.launchPuppeteer({ proxyUrl: 'socks4://example.com:1234' });
- browser1.close();
+ closePromises.push(browser1.close());
const browser2 = await Apify.launchPuppeteer({ proxyUrl: 'socks5://example.com:1234' });
- browser2.close();
+ closePromises.push(browser2.close());
const browser3 = await Apify.launchPuppeteer({ proxyUrl: 'https://example.com:1234' });
- browser3.close();
+ closePromises.push(browser3.close());
const browser4 = await Apify.launchPuppeteer({ proxyUrl: 'HTTP://example.com:1234' });
- browser4.close();
+ closePromises.push(browser4.close());
+ await Promise.all(closePromises);
});
test('opens https://www.example.com', () => {
diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.js
index 46aa9b1b7..59a00f907 100644
--- a/test/crawlers/cheerio_crawler.test.js
+++ b/test/crawlers/cheerio_crawler.test.js
@@ -650,7 +650,7 @@ describe('CheerioCrawler', () => {
let requestList;
beforeEach(async () => {
- requestList = await Apify.openRequestList('test', sources.slice());
+ requestList = await Apify.openRequestList(null, sources.slice());
});
test('should work', async () => {
@@ -685,10 +685,8 @@ describe('CheerioCrawler', () => {
test('should markBad sessions after request timeout', async () => {
log.setLevel(log.LEVELS.OFF);
- const sessions = [];
- const failed = [];
const cheerioCrawler = new Apify.CheerioCrawler({
- requestList: await Apify.openRequestList(`timeoutTest-${Math.random()}`, [
+ requestList: await Apify.openRequestList(null, [
`http://${HOST}:${port}/timeout?a=12`,
`http://${HOST}:${port}/timeout?a=23`,
]),
@@ -697,15 +695,11 @@ describe('CheerioCrawler', () => {
maxConcurrency: 1,
useSessionPool: true,
handlePageFunction: async () => {},
- handleFailedRequestFunction: ({ request }) => failed.push(request),
});
- const oldCall = cheerioCrawler._handleRequestTimeout.bind(cheerioCrawler);
- cheerioCrawler._handleRequestTimeout = (session) => {
- sessions.push(session);
- return oldCall(session);
- };
await cheerioCrawler.run();
+
+ const { sessions } = cheerioCrawler.sessionPool;
expect(sessions.length).toBe(4);
sessions.forEach((session) => {
expect(session.errorScore).toEqual(1);