Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Change back to first page, because it needs to be in focus
Ondra Urban357423f8ee4f
diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.jsindex 53667e0e4..76206c90f 100644--- a/src/live_view/live_view_server.js+++ b/src/live_view/live_view_server.js@@ -78,6 +78,7 @@ class LiveViewServer {this._isRunning = false;this.httpServer = null;this.socketio = null;+ this.servingSnapshot = false;this._setupHttpServer();}@@ -120,9 +121,18 @@ class LiveViewServer {* @return {Promise}*/async serve(page) {+ // Only serve one snapshot at a time because Puppeteer+ // can't make screenshots in parallel.+ if (this.servingSnapshot) return;if (!this.hasClients()) return;- const snapshot = await this._makeSnapshot(page);- await this._pushSnapshot(snapshot);++ try {+ this.servingSnapshot = true;+ const snapshot = await this._makeSnapshot(page);+ await this._pushSnapshot(snapshot);+ } finally {+ this.servingSnapshot = false;+ }}/**@@ -154,7 +164,7 @@ class LiveViewServer {* @ignore*/_getScreenshotPath(screenshotIndex) {- return path.join(this.screenshotDirectoryPath, `${screenshotIndex}.png`);+ return path.join(this.screenshotDirectoryPath, `${screenshotIndex}.jpeg`);}async _makeSnapshot(page) {@@ -162,7 +172,11 @@ class LiveViewServer {log.info('Making live view snapshot.', { pageUrl });const [htmlContent, screenshot] = await Promise.all([page.content(),- page.screenshot(),+ page.screenshot({+ type: 'jpeg',+ quality: 75,+ fullPage: true,+ }),]);const screenshotIndex = this.lastScreenshotIndex++;diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 2ea8f83c8..bf71953be 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -15,7 +15,6 @@ import LiveViewServer from './live_view/live_view_server';const PROCESS_KILL_TIMEOUT_MILLIS = 5000;const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;-const LIVE_VIEW_SNAPSHOT_MAX_AGE_MILLIS = 2000;const LIVE_VIEW_SNAPSHOT_TIMEOUT_MILLIS = 5000;const DEFAULT_OPTIONS = {@@ -657,7 +656,7 @@ class PuppeteerPool {* @return {Promise}*/async recyclePage(page) {- if (this.liveViewServer && this.liveViewServer.hasClients()) {+ if (this.liveViewServer) {await this._serveLiveView(page).catch(err => log.exception(err, 'Live View failed to be served.'));}@@ -678,14 +677,13 @@ class PuppeteerPool {}async _serveLiveView(page) {+ if (!this.liveViewServer.hasClients()) return;const browser = page.browser();- const lastSnapshot = this.liveViewServer.getLastSnapshot();+ const pages = await browser.pages();- // Only make new snapshot after some time to prevent overloading.- if (lastSnapshot) {- const millisSinceLastSnapshot = Date.now() - lastSnapshot.createdAt;- if (millisSinceLastSnapshot < LIVE_VIEW_SNAPSHOT_MAX_AGE_MILLIS) return;- }+ // We only serve the second page of the browser because it's in focus,+ // which is necessary for screenshots. First page is about:blank.+ if (pages[1] !== page) return;const instance = await this._findInstanceByBrowser(browser);// Sometimes the browser gets killed and there's no instance.diff --git a/test/live_view/live_view_server.js b/test/live_view/live_view_server.jsindex 3a3cdc464..472270f36 100644--- a/test/live_view/live_view_server.js+++ b/test/live_view/live_view_server.js@@ -117,7 +117,9 @@ describe('LiveViewServer', () => {it('should not store more than maxScreenshotFiles screenshots', async () => {const snapshots = [];socket.on('snapshot', s => snapshots.push(s));- await Promise.all(Array(5).fill(null).map(() => lvs.serve(fakePage)));+ for (let i = 0; i < 5; i++) {+ await lvs.serve(fakePage);+ }const files = await new Promise((resolve, reject) => {const interval = setInterval(async () => {const files = await readdir(LOCAL_STORAGE_SUBDIR); // eslint-disable-line no-shadow@@ -131,7 +133,7 @@ describe('LiveViewServer', () => {reject(new Error('Files were not deleted in 2000ms.'));}, 2000);});- files.forEach((f, idx) => expect(f).to.be.eql(`${idx + 3}.png`));+ files.forEach((f, idx) => expect(f).to.be.eql(`${idx + 3}.jpeg`));});});});