Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Finalize LiveViewServer tests

Ondra Urban53299d9bc52b
diff --git a/src/live_view/index.html b/src/live_view/index.html
index 01a16ab1c..d55ea06b2 100644
--- a/src/live_view/index.html
+++ b/src/live_view/index.html
@@ -22,7 +22,7 @@
if (!snapshot) return;
$('#pageUrl').text(snapshot.pageUrl).attr('href', snapshot.pageUrl);
$('#htmlContent').text(snapshot.htmlContent);
- $('#screenshotImg').attr('src', `./screenshot?index=${snapshot.screenshotIndex}`);
+ $('#screenshotImg').attr('src', `./screenshot/${snapshot.screenshotIndex}`);
$('#waitContainer').hide();
$('#snapshotContainer').show();
Prism.highlightAll();
diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.js
index 01b6820dd..a8a16b958 100644
--- a/src/live_view/live_view_server.js
+++ b/src/live_view/live_view_server.js
@@ -5,11 +5,12 @@ import { promisify } from 'util';
import express from 'express';
import socketio from 'socket.io';
import log from 'apify-shared/log';
+import { checkParamOrThrow } from 'apify-client/build/utils';
import { promisifyServerListen } from 'apify-shared/utilities';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
const DEFAULT_SCREENSHOT_DIRECTORY_PATH = path.resolve('live_view');
-const MAX_SCREENSHOT_FILES = 10;
+const DEFAULT_MAX_SCREENSHOT_FILES = 10;
const writeFile = promisify(fs.writeFile);
@@ -20,6 +21,7 @@ export default class LiveViewServer {
constructor(options = {}) {
const {
screenshotDirectoryPath = DEFAULT_SCREENSHOT_DIRECTORY_PATH,
+ maxScreenshotFiles = DEFAULT_MAX_SCREENSHOT_FILES,
} = options;
const containerPort = process.env[ENV_VARS.CONTAINER_PORT] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_PORT];
@@ -30,12 +32,15 @@ export default class LiveViewServer {
+ `${ENV_VARS.CONTAINER_PORT} environment variable (was "${containerPort}").`);
}
this.liveViewUrl = process.env[ENV_VARS.CONTAINER_URL] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_URL];
+
+ checkParamOrThrow(screenshotDirectoryPath, 'options.screenshotDirectoryPath', 'String');
+ checkParamOrThrow(maxScreenshotFiles, 'options.maxScreenshotFiles', 'Number');
this.screenshotDirectoryPath = screenshotDirectoryPath;
+ this.maxScreenshotFiles = maxScreenshotFiles;
// Snapshot data
this.lastSnapshot = null;
this.lastScreenshotIndex = 0;
- this.screenshotIndexToFilePath = {};
// Setup HTTP server and Express router
this._isRunning = false;
@@ -51,14 +56,9 @@ export default class LiveViewServer {
});
// Serves JPEG with the last screenshot
- this.app.get('/screenshot', (req, res) => {
- const screenshotIndex = req.query.index;
- const filePath = this.screenshotIndexToFilePath[screenshotIndex];
- if (!filePath) {
- return res.status(404).send('Oops, there is no such screenshot.');
- }
- // TODO: Limit snapshot sizes to avoid choking the system (was 5MB),
- // return replacement image if too large
+ this.app.get('/screenshot/:index', (req, res) => {
+ const screenshotIndex = req.params.index;
+ const filePath = this.getScreenshotPath(screenshotIndex);
res.sendFile(filePath);
});
@@ -117,7 +117,7 @@ export default class LiveViewServer {
}
getScreenshotPath(screenshotIndex) {
- return path.join(this.screenshotDirectoryPath, screenshotIndex);
+ return path.join(this.screenshotDirectoryPath, `${screenshotIndex}`);
}
async _makeSnapshot(page) {
@@ -131,8 +131,8 @@ export default class LiveViewServer {
const screenshotIndex = this.lastScreenshotIndex++;
await writeFile(this.getScreenshotPath(screenshotIndex), screenshot);
- if (screenshotIndex > MAX_SCREENSHOT_FILES) {
- this._deleteScreenshot(MAX_SCREENSHOT_FILES - screenshotIndex);
+ if (screenshotIndex > this.maxScreenshotFiles - 1) {
+ this._deleteScreenshot(screenshotIndex - this.maxScreenshotFiles);
}
const snapshot = { pageUrl, htmlContent, screenshotIndex };
diff --git a/test/live_view/live_view_server.js b/test/live_view/live_view_server.js
index b0fe4bc01..d9aad1e45 100644
--- a/test/live_view/live_view_server.js
+++ b/test/live_view/live_view_server.js
@@ -1,12 +1,21 @@
import path from 'path';
+import { promisify } from 'util';
+import rqst from 'request-promise-native';
+import fs from 'fs-extra';
import { expect } from 'chai';
+import io from 'socket.io-client'; // eslint-disable-line
+import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import Apify from '../../build/index';
import { LOCAL_STORAGE_DIR } from '../_helper';
import LiveViewServer from '../../build/live_view/live_view_server';
const { utils: { log } } = Apify;
+const emptyDir = promisify(fs.emptyDir);
+const readdir = promisify(fs.readdir);
const LOCAL_STORAGE_SUBDIR = path.join(LOCAL_STORAGE_DIR, 'live_view');
+const PORT = LOCAL_ENV_VARS[ENV_VARS.CONTAINER_PORT];
+const BASE_URL = `http://localhost:${PORT}`;
let originalLogLevel;
before(() => {
@@ -19,19 +28,97 @@ after(() => {
});
describe('LiveViewServer', () => {
- it('should construct', () => {
- const lvs = new LiveViewServer({
+ let lvs;
+ beforeEach(() => {
+ lvs = new LiveViewServer({
screenshotDirectoryPath: LOCAL_STORAGE_SUBDIR,
+ maxScreenshotFiles: 2,
});
+ });
+
+ afterEach(async () => {
+ lvs = null;
+ await emptyDir(LOCAL_STORAGE_SUBDIR);
+ });
+
+ it('should construct', async () => {
expect(lvs).to.be.instanceOf(LiveViewServer);
});
+
it('should start and stop', async () => {
- const lvs = new LiveViewServer({
- screenshotDirectoryPath: LOCAL_STORAGE_SUBDIR,
- });
await lvs.start();
expect(lvs.isRunning).to.be.eql(true);
await lvs.stop();
expect(lvs.isRunning).to.be.eql(false);
});
+
+ it('should connect over websocket', async () => {
+ const socket = io(BASE_URL);
+ const connected = new Promise((resolve) => {
+ socket.on('connect', async () => {
+ expect(lvs.hasClients()).to.be.eql(true);
+ await lvs.stop();
+ socket.close();
+ resolve();
+ });
+ });
+ await lvs.start();
+ await connected;
+ });
+
+ describe('when connected', () => {
+ const fakePage = {
+ url: () => 'url',
+ content: async () => 'content',
+ screenshot: async () => `screenshot${count++}`,
+ };
+ let count;
+ let socket;
+ beforeEach(async () => {
+ count = 0;
+ socket = io(BASE_URL);
+ await lvs.start();
+ await new Promise(resolve => socket.on('connect', resolve));
+ });
+ afterEach(async () => {
+ socket.close();
+ await lvs.stop();
+ count = null;
+ socket = null;
+ });
+
+ it('should serve snapshot', async () => {
+ await lvs.serve(fakePage);
+ const snapshot = await new Promise(resolve => socket.on('snapshot', resolve));
+ expect(snapshot).to.be.eql({ pageUrl: 'url', htmlContent: 'content', screenshotIndex: 0 });
+ });
+
+ it('should return screenshots', async () => {
+ const snapshot0 = new Promise(resolve => socket.on('snapshot', resolve));
+ await lvs.serve(fakePage);
+ const response0 = await rqst(`${BASE_URL}/screenshot/${((await snapshot0).screenshotIndex)}`);
+ expect(response0).to.be.eql('screenshot0');
+ const snapshot1 = new Promise(resolve => socket.on('snapshot', resolve));
+ await lvs.serve(fakePage);
+ const response1 = await rqst(`${BASE_URL}/screenshot/${(await snapshot1).screenshotIndex}`);
+ expect(response1).to.be.eql('screenshot1');
+ });
+
+ it('should not store more than maxScreenshotFiles screenshots', async () => {
+ const snapshots = [];
+ socket.on('snapshot', s => snapshots.push(s));
+ await Promise.all(Array(3).fill(null).map(_ => lvs.serve(fakePage)));
+ const screenshots = await Promise.all(snapshots
+ .sort((a, b) => a.screenshotIndex - b.screenshotIndex)
+ .map((snap, idx) => {
+ expect(snap.screenshotIndex).to.be.eql(idx);
+ return snap;
+ })
+ .map(({ screenshotIndex }) => rqst(`${BASE_URL}/screenshot/${screenshotIndex}`)));
+ screenshots.forEach((s, idx) => expect(s).to.be.eql(`screenshot${idx}`));
+ const files = await readdir(LOCAL_STORAGE_SUBDIR);
+ expect(files).to.have.lengthOf(2);
+ files.forEach((f, idx) => expect(f).to.be.eql(`${idx + 1}`));
+ });
+ });
});