Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix/save snapshot file format (#651)

Jan Bárta91e0d3e40564
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 79a296219..888ad4438 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -6,6 +6,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';
import * as LruCache from 'apify-shared/lru_cache';
import { Page, Response, DirectNavigationOptions } from 'puppeteer'; // eslint-disable-line no-unused-vars
+import { isAtHome } from './utils';
import log from './utils_log';
import { RequestQueue, RequestQueueLocal } from './request_queue';
@@ -545,12 +546,14 @@ const saveSnapshot = async (page, options = {}) => {
const store = await openKeyValueStore(keyValueStoreName);
if (saveScreenshot) {
+ const screenshotName = isAtHome() ? `${key}.jpg` : key;
const screenshotBuffer = await page.screenshot({ fullPage: true, screenshotQuality, type: 'jpeg' });
- await store.setValue(`${key}.jpg`, screenshotBuffer, { contentType: 'image/jpeg' });
+ await store.setValue(screenshotName, screenshotBuffer, { contentType: 'image/jpeg' });
}
if (saveHtml) {
+ const htmlName = isAtHome() ? `${key}.html` : key;
const html = await page.content();
- await store.setValue(`${key}.html`, html, { contentType: 'text/html' });
+ await store.setValue(htmlName, html, { contentType: 'text/html' });
}
} catch (e) {
// I like this more than having to investigate stack trace
diff --git a/test/puppeteer_utils.test.js b/test/puppeteer_utils.test.js
index e24241356..98d7c9758 100644
--- a/test/puppeteer_utils.test.js
+++ b/test/puppeteer_utils.test.js
@@ -1,5 +1,6 @@
import sinon from 'sinon';
import path from 'path';
+import { ENV_VARS } from 'apify-shared/consts';
import Apify from '../build/index';
import * as keyValueStore from '../build/key_value_store';
import LocalStorageDirEmulator from './local_storage_dir_emulator';
@@ -375,8 +376,8 @@ describe('Apify.utils.puppeteer', () => {
await Apify.utils.puppeteer.saveSnapshot(page, { key: 'TEST', keyValueStoreName: 'TEST-STORE', screenshotQuality: 60 });
- expect(stub.calledWithExactly('TEST.jpg', screenshot, { contentType: 'image/jpeg' })).toBe(true);
- expect(stub.calledWithExactly('TEST.html', contentHTML, { contentType: 'text/html' })).toBe(true);
+ expect(stub.calledWithExactly('TEST', screenshot, { contentType: 'image/jpeg' })).toBe(true);
+ expect(stub.calledWithExactly('TEST', contentHTML, { contentType: 'text/html' })).toBe(true);
// Test saving only image
const object2 = { setValue: async () => {} };
@@ -389,7 +390,23 @@ describe('Apify.utils.puppeteer', () => {
// Default quality is 50
const screenshot2 = await page.screenshot({ fullPage: true, type: 'jpeg', screenshotQuality: 50 });
- expect(stub2.calledOnceWithExactly('SNAPSHOT.jpg', screenshot2, { contentType: 'image/jpeg' })).toBe(true);
+ expect(stub2.calledOnceWithExactly('SNAPSHOT', screenshot2, { contentType: 'image/jpeg' })).toBe(true);
+
+ // Test saving both image and html on the platform
+ process.env[ENV_VARS.IS_AT_HOME] = 1;
+
+ const object3 = { setValue: async () => {} };
+ const stub3 = sinon.stub(object3, 'setValue');
+ mock.expects('openKeyValueStore')
+ .once()
+ .withExactArgs('TEST-STORE')
+ .resolves(object3);
+
+ await Apify.utils.puppeteer.saveSnapshot(page, { key: 'TEST', keyValueStoreName: 'TEST-STORE', screenshotQuality: 60 });
+
+ expect(stub3.calledWithExactly('TEST.jpg', screenshot, { contentType: 'image/jpeg' })).toBe(true);
+ expect(stub3.calledWithExactly('TEST.html', contentHTML, { contentType: 'text/html' })).toBe(true);
+ delete process.env[ENV_VARS.IS_AT_HOME];
mock.verify();
} finally {