Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Improve UX
Ondra Urban566fdb24db7d
diff --git a/package.json b/package.jsonindex 69668cee6..ec7a7b95c 100644--- a/package.json+++ b/package.json@@ -38,7 +38,7 @@"build"],"scripts": {- "build": "rm -rf ./build && babel src --out-dir build",+ "build": "rm -rf ./build && babel src --out-dir build --copy-files","build-docs": "npm run clean && npm run build && node ./website/tools/build_docs.js","build-readme": "node ./tools/build_readme.js","test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --require @babel/register --recursive --exit",diff --git a/src/live_view/index.html b/src/live_view/index.htmlindex d55ea06b2..bdb4dabe2 100644--- a/src/live_view/index.html+++ b/src/live_view/index.html@@ -12,6 +12,17 @@.urlDiv {margin-bottom: 8px;}+ button {+ color: white;+ border-radius: 4px;+ text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2);+ }+ #enableButton {+ background: rgb(28, 184, 65);+ }+ #disableButton {+ background: rgb(202, 60, 60);+ }</style><script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.2.0/socket.io.slim.js"></script><script src="https://code.jquery.com/jquery-1.11.1.min.js"></script>@@ -27,20 +38,37 @@$('#snapshotContainer').show();Prism.highlightAll();}- $(function () {- var socket = io();+ function connect() {+ const socket = io();socket.on('snapshot', showSnapshot);- socket.on('connect', function() {+ socket.on('connect', () => {console.log('Client ID: ' + socket.id);});+ return socket;+ }+ $(() => {+ let socket;// Show initial snapshot obtained from /init-last-snapshot.jsshowSnapshot(window.lastSnapshot);- $('#screenshotLink').click(function (event) {+ $('#enableButton').click((event) => {+ event.preventDefault();+ socket = connect();+ $('#waitContainer').show();+ $('#enableButton').hide();+ $('#disableButton').show();+ });+ $('#disableButton').click((event) => {+ event.preventDefault();+ if (socket) socket.close();+ $('#enableButton').show();+ $('#disableButton').hide();+ });+ $('#screenshotLink').click((event) => {event.preventDefault();$('#screenshotDiv').show();$('#htmlDiv').hide();});- $('#htmlLink').click(function () {+ $('#htmlLink').click(() => {event.preventDefault();$('#screenshotDiv').hide();$('#htmlDiv').show();@@ -52,7 +80,12 @@<script src="./prism.js"></script>-<p id="waitContainer">+<header id="header">+ <button id="enableButton">Enable Live View</button>+ <button id="disableButton" style="display: none">Disable Live View</button>+</header>++<p id="waitContainer" style="display: none">Waiting for the snapshot...<br><img src="./loading.gif"/></p>diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.jsindex 3de5c7670..84c8cb653 100644--- a/src/live_view/live_view_server.js+++ b/src/live_view/live_view_server.js@@ -24,15 +24,6 @@ export default class LiveViewServer {maxScreenshotFiles = DEFAULT_MAX_SCREENSHOT_FILES,} = options;- const containerPort = process.env[ENV_VARS.CONTAINER_PORT] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_PORT];-- this.port = parseInt(containerPort, 10);- if (!(this.port >= 0 && this.port <= 65535)) {- throw new Error('Cannot start LiveViewServer - invalid port specified by the '- + `${ENV_VARS.CONTAINER_PORT} environment variable (was "${containerPort}").`);- }- this.liveViewUrl = process.env[ENV_VARS.CONTAINER_URL] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_URL];-checkParamOrThrow(screenshotDirectoryPath, 'options.screenshotDirectoryPath', 'String');checkParamOrThrow(maxScreenshotFiles, 'options.maxScreenshotFiles', 'Number');this.screenshotDirectoryPath = screenshotDirectoryPath;@@ -42,43 +33,13 @@ export default class LiveViewServer {this.lastSnapshot = null;this.lastScreenshotIndex = 0;- // Setup HTTP server and Express router+ // Server+ this.clientCount = 0;this._isRunning = false;- this.httpServer = http.createServer();- this.app = express();-- this.app.use('/', express.static(__dirname));-- // Serves a JS file with the last snapshot, so the the client can immediately show something- this.app.get('/init-last-snapshot.js', (req, res) => {- res.set('Content-Type', 'text/javascript');- res.send(`window.lastSnapshot = ${JSON.stringify(this.lastSnapshot, null, 2)};`);- });+ this.httpServer = null;+ this.socketio = null;- // Serves JPEG with the last screenshot- this.app.get('/screenshot/:index', (req, res) => {- const screenshotIndex = req.params.index;- const filePath = this.getScreenshotPath(screenshotIndex);- res.sendFile(filePath);- });-- this.app.all('*', (req, res) => {- res.status(404).send('Nothing here');- });-- this.httpServer.on('request', this.app);-- // Socket.io server used to send snapshots to client- this.clientCount = 0;- this.socketio = socketio(this.httpServer);- this.socketio.on('connection', (socket) => {- this.clientCount++;- log.info('Live view client connected', { clientId: socket.id });- socket.on('disconnect', (reason) => {- this.clientCount--;- log.info('Live view client disconnected', { clientId: socket.id, reason });- });- });+ this._setupHttpServer();}/**@@ -146,7 +107,7 @@ export default class LiveViewServer {*/async _pushSnapshot(snapshot) {// Send new snapshot to clients- log.info('Sending live view snapshot', { snapshot });+ log.debug('Sending live view snapshot', { snapshot });this.socketio.emit('snapshot', snapshot);}@@ -154,4 +115,50 @@ export default class LiveViewServer {unlink(this.getScreenshotPath(screenshotIndex)).catch(err => log.exception(err, 'Cannot delete live view screenshot.'));}++ _setupHttpServer() {+ const containerPort = process.env[ENV_VARS.CONTAINER_PORT] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_PORT];++ this.port = parseInt(containerPort, 10);+ if (!(this.port >= 0 && this.port <= 65535)) {+ throw new Error('Cannot start LiveViewServer - invalid port specified by the '+ + `${ENV_VARS.CONTAINER_PORT} environment variable (was "${containerPort}").`);+ }+ this.liveViewUrl = process.env[ENV_VARS.CONTAINER_URL] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_URL];++ this.httpServer = http.createServer();+ const app = express();++ app.use('/', express.static(__dirname));++ // Serves a JS file with the last snapshot, so the the client can immediately show something+ app.get('/init-last-snapshot.js', (req, res) => {+ res.set('Content-Type', 'text/javascript');+ res.send(`window.lastSnapshot = ${JSON.stringify(this.lastSnapshot, null, 2)};`);+ });++ // Serves JPEG with the last screenshot+ app.get('/screenshot/:index', (req, res) => {+ const screenshotIndex = req.params.index;+ const filePath = this.getScreenshotPath(screenshotIndex);+ res.sendFile(filePath);+ });++ app.all('*', (req, res) => {+ res.status(404).send('Nothing here');+ });++ this.httpServer.on('request', app);++ // Socket.io server used to send snapshots to client+ this.socketio = socketio(this.httpServer);+ this.socketio.on('connection', (socket) => {+ this.clientCount++;+ log.info('Live view client connected', { clientId: socket.id });+ socket.on('disconnect', (reason) => {+ this.clientCount--;+ log.info('Live view client disconnected', { clientId: socket.id, reason });+ });+ });+ }}diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 8a491015d..79a20dbd1 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -653,7 +653,7 @@ class PuppeteerPool {* @return {Promise}*/async recyclePage(page) {- if (this.liveViewServer) {+ if (this.liveViewServer && this.liveViewServer.hasClients()) {await this._serveLiveView(page).catch(err => log.exception(err, 'LiveView failed to be served.'));}@@ -675,7 +675,16 @@ class PuppeteerPool {async _serveLiveView(page) {const browser = page.browser();+ const pages = await browser.pages();++ // We only serve the second page of the browser.+ // First page is about:blank and the others are disregarded.+ if (pages[1] !== page) return;+const instance = await this._findInstanceByBrowser(browser);+ // Sometimes the browser gets killed and there's no instance.+ if (!instance) return;+// Only take snapshots in the most recently opened browser.if (instance.id !== this.browserCounter - 1) return;await this.liveViewServer.serve(page);diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex 51bc58159..616f16991 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -594,6 +594,7 @@ describe('PuppeteerPool', () => {pool.liveViewServer.start = async () => {started++;pool.liveViewServer._isRunning = true; // eslint-disable-line no-underscore-dangle+ pool.liveViewServer.clientCount++;};pool.liveViewServer.serve = async arg => serveCalledWith.push(arg);pool.liveViewServer.stop = async () => stopped++;