Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Tiny refactoring and fixing tests

Marek Trunkat68335617b789
diff --git a/src/puppeteer_live_view_server.js b/src/puppeteer_live_view_server.js
index b94af5f8e..3abb04ac4 100644
--- a/src/puppeteer_live_view_server.js
+++ b/src/puppeteer_live_view_server.js
@@ -2,6 +2,7 @@ import http from 'http';
import EventEmitter from 'events';
import Promise from 'bluebird';
import WebSocket from 'ws';
+import _ from 'underscore';
import log from 'apify-shared/log';
import { promisifyServerListen } from 'apify-shared/utilities';
import { checkParamOrThrow } from 'apify-client/build/utils';
@@ -279,6 +280,19 @@ const sendCommand = (socket, command, data) => {
export default class PuppeteerLiveViewServer extends EventEmitter {
constructor() {
super();
+
+ this.containerUrl = process.env[ENV_VARS.CONTAINER_URL] || 'http://localhost';
+ this.containerPort = parseInt(process.env[ENV_VARS.CONTAINER_PORT], 10) || 0;
+
+ if (!this.containerUrl) {
+ throw new Error(`Cannot start LiveViewServer - the environment variable ${ENV_VARS.CONTAINER_URL} is not set!
+If you are running on localhost then set ${ENV_VARS.CONTAINER_URL}='http://localhost'`);
+ }
+
+ if (!_.isNumber(this.containerPort)) {
+ throw new Error(`Cannot start LiveViewServer - the environment variable ${ENV_VARS.CONTAINER_PORT} is not set!`);
+ }
+
this.browsers = new Set();
this.browserIdCounter = 0;
this.httpServer = null;
@@ -313,20 +327,14 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
* @return {Promise} resolves when HTTP server starts listening
*/
startServer() {
- const getRejection = envVar =>
- Promise.reject(new Error(`The environment variable ${envVar} is not set. 'LiveViewServer cannot be started.`));
-
- if (!process.env[ENV_VARS.CONTAINER_URL]) return getRejection(ENV_VARS.CONTAINER_URL);
-
- const port = process.env[ENV_VARS.CONTAINER_PORT];
- if (port == null || Number.isNaN(port)) return getRejection(ENV_VARS.CONTAINER_PORT);
-
const server = http.createServer(this._httpRequestListener.bind(this));
const wss = new WebSocket.Server({ server });
+
wss.on('connection', this._wsRequestListener.bind(this));
- return promisifyServerListen(server)(port)
+
+ return promisifyServerListen(server)(this.containerPort)
.then(() => {
- log.info(`Live view server is listening on port ${server.address().port}.`);
+ log.info(`Live view server is now avilable at URL ${this.containerUrl} (running on port ${server.address().port}).`);
this.httpServer = server;
});
}
@@ -341,7 +349,7 @@ export default class PuppeteerLiveViewServer extends EventEmitter {
* @ignore
*/
_httpRequestListener(req, res) {
- const body = layout(process.env[ENV_VARS.CONTAINER_URL]);
+ const body = layout(this.containerUrl);
res.writeHead(200, {
'Content-Type': 'text/html',
'Content-Length': Buffer.byteLength(body),
diff --git a/test/puppeteer_live_view_server.js b/test/puppeteer_live_view_server.js
index 68fa55305..43a8f3d97 100644
--- a/test/puppeteer_live_view_server.js
+++ b/test/puppeteer_live_view_server.js
@@ -31,16 +31,20 @@ beforeEach(() => {
process.env[ENV_VARS.CONTAINER_URL] = URL;
});
-
describe('Starting the PuppeteerLiveViewServer', () => {
- it('should start using Apify.launchPuppeteer()', () => {
+ it('should start using Apify.launchPuppeteer()', async () => {
const customPort = 1234;
+
process.env[ENV_VARS.CONTAINER_PORT] = customPort;
+
return Apify.launchPuppeteer({ liveView: true })
- .then(() => httpGet(customPort))
- .catch(err => assert.fail(err, 'Server response.'))
- .then((body) => {
- expect(body.trim().substr(0, 15)).to.equal('<!doctype html>');
+ .then((browser) => {
+ return httpGet(customPort)
+ .catch(err => assert.fail(err, 'Server response.'))
+ .then((body) => {
+ expect(body.trim().substr(0, 15)).to.equal('<!doctype html>');
+ })
+ .then(() => browser.close());
});
});
it('should start using PuppeteerLiveViewServer.startServer()', () => {
@@ -82,14 +86,22 @@ describe('Manipulate the PuppeteerLiveViewServer', () => {
destroyHandler++;
});
+ const browsers = [];
+
return puppeteer.launch()
- .then(browser => server.addBrowser(new PuppeteerLiveViewBrowser(browser, { id: 'B1' })))
+ .then((browser) => {
+ browsers.push(browser);
+ return server.addBrowser(new PuppeteerLiveViewBrowser(browser, { id: 'B1' }));
+ })
.then(() => {
expect(server.browsers.size).to.equal(1);
expect(createHandler).to.equal(1);
})
.then(() => puppeteer.launch())
- .then(browser => server.addBrowser(new PuppeteerLiveViewBrowser(browser, { id: 'B2' })))
+ .then((browser) => {
+ browsers.push(browser);
+ return server.addBrowser(new PuppeteerLiveViewBrowser(browser, { id: 'B2' }));
+ })
.then(() => {
expect(server.browsers.size).to.equal(2);
expect(createHandler).to.equal(2);
@@ -99,6 +111,7 @@ describe('Manipulate the PuppeteerLiveViewServer', () => {
expect(server.browsers.size).to.equal(1);
expect(createHandler).to.equal(2);
expect(destroyHandler).to.equal(1);
- });
+ })
+ .then(() => Promise.all(browsers.map(browser => browser.close())));
});
});