Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix/save snapshot file format (#651)
Jan Bárta91e0d3e40564
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 79a296219..888ad4438 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -6,6 +6,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';import * as LruCache from 'apify-shared/lru_cache';import { Page, Response, DirectNavigationOptions } from 'puppeteer'; // eslint-disable-line no-unused-vars+import { isAtHome } from './utils';import log from './utils_log';import { RequestQueue, RequestQueueLocal } from './request_queue';@@ -545,12 +546,14 @@ const saveSnapshot = async (page, options = {}) => {const store = await openKeyValueStore(keyValueStoreName);if (saveScreenshot) {+ const screenshotName = isAtHome() ? `${key}.jpg` : key;const screenshotBuffer = await page.screenshot({ fullPage: true, screenshotQuality, type: 'jpeg' });- await store.setValue(`${key}.jpg`, screenshotBuffer, { contentType: 'image/jpeg' });+ await store.setValue(screenshotName, screenshotBuffer, { contentType: 'image/jpeg' });}if (saveHtml) {+ const htmlName = isAtHome() ? `${key}.html` : key;const html = await page.content();- await store.setValue(`${key}.html`, html, { contentType: 'text/html' });+ await store.setValue(htmlName, html, { contentType: 'text/html' });}} catch (e) {// I like this more than having to investigate stack tracediff --git a/test/puppeteer_utils.test.js b/test/puppeteer_utils.test.jsindex e24241356..98d7c9758 100644--- a/test/puppeteer_utils.test.js+++ b/test/puppeteer_utils.test.js@@ -1,5 +1,6 @@import sinon from 'sinon';import path from 'path';+import { ENV_VARS } from 'apify-shared/consts';import Apify from '../build/index';import * as keyValueStore from '../build/key_value_store';import LocalStorageDirEmulator from './local_storage_dir_emulator';@@ -375,8 +376,8 @@ describe('Apify.utils.puppeteer', () => {await Apify.utils.puppeteer.saveSnapshot(page, { key: 'TEST', keyValueStoreName: 'TEST-STORE', screenshotQuality: 60 });- expect(stub.calledWithExactly('TEST.jpg', screenshot, { contentType: 'image/jpeg' })).toBe(true);- expect(stub.calledWithExactly('TEST.html', contentHTML, { contentType: 'text/html' })).toBe(true);+ expect(stub.calledWithExactly('TEST', screenshot, { contentType: 'image/jpeg' })).toBe(true);+ expect(stub.calledWithExactly('TEST', contentHTML, { contentType: 'text/html' })).toBe(true);// Test saving only imageconst object2 = { setValue: async () => {} };@@ -389,7 +390,23 @@ describe('Apify.utils.puppeteer', () => {// Default quality is 50const screenshot2 = await page.screenshot({ fullPage: true, type: 'jpeg', screenshotQuality: 50 });- expect(stub2.calledOnceWithExactly('SNAPSHOT.jpg', screenshot2, { contentType: 'image/jpeg' })).toBe(true);+ expect(stub2.calledOnceWithExactly('SNAPSHOT', screenshot2, { contentType: 'image/jpeg' })).toBe(true);++ // Test saving both image and html on the platform+ process.env[ENV_VARS.IS_AT_HOME] = 1;++ const object3 = { setValue: async () => {} };+ const stub3 = sinon.stub(object3, 'setValue');+ mock.expects('openKeyValueStore')+ .once()+ .withExactArgs('TEST-STORE')+ .resolves(object3);++ await Apify.utils.puppeteer.saveSnapshot(page, { key: 'TEST', keyValueStoreName: 'TEST-STORE', screenshotQuality: 60 });++ expect(stub3.calledWithExactly('TEST.jpg', screenshot, { contentType: 'image/jpeg' })).toBe(true);+ expect(stub3.calledWithExactly('TEST.html', contentHTML, { contentType: 'text/html' })).toBe(true);+ delete process.env[ENV_VARS.IS_AT_HOME];mock.verify();} finally {