Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

enhanced options for utils.puppeteer.saveSnapshot

metalwarrior665b25c8692bb71
diff --git a/docs/api/puppeteer.md b/docs/api/puppeteer.md
index 668b15278..28506b980 100644
--- a/docs/api/puppeteer.md
+++ b/docs/api/puppeteer.md
@@ -28,6 +28,7 @@ await puppeteer.injectJQuery(page);
- [`.removeInterceptRequestHandler`](#puppeteer.removeInterceptRequestHandler) ⇒ `Promise`
- [`.gotoExtended`](#puppeteer.gotoExtended) ⇒ `Promise<Response>`
- [`.infiniteScroll`](#puppeteer.infiniteScroll) ⇒ `Promise`
+ - [`.saveSnapshot`](#puppeteer.saveSnapshot) ⇒ `Promise`
- [`.injectFile(page, filePath, [options])`](#puppeteer.injectFile) ⇒ `Promise`
- [`.injectJQuery(page)`](#puppeteer.injectJQuery) ⇒ `Promise`
- [`.injectUnderscore(page)`](#puppeteer.injectUnderscore) ⇒ `Promise`
@@ -320,6 +321,51 @@ Scrolls to the bottom of a page, or until it times out. Loads dynamic content wh
<td colspan="3"><p>How many seconds to wait for no new content to load before exit.</p>
</td></tr></tbody>
</table>
+<a name="puppeteer.saveSnapshot"></a>
+
+## `puppeteer.saveSnapshot` ⇒ `Promise`
+
+Saves a full screenshot and HTML of the current page into a Key-Value store.
+
+<table>
+<thead>
+<tr>
+<th>Param</th><th>Type</th><th>Default</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>page</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"><p>Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.</p>
+</td></tr><tr>
+<td><code>[options]</code></td><td><code>Object</code></td><td></td>
+</tr>
+<tr>
+<td colspan="3"></td></tr><tr>
+<td><code>[options.key]</code></td><td><code>String</code></td><td><code>SNAPSHOT</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Key under which the screenshot and HTML will be saved. <code>.png</code> will be appended for screenshot and <code>.html</code> for HTML.
+ Must contain only letters, numbers, dashes, dots and underscores.</p>
+</td></tr><tr>
+<td><code>[options.saveScreenshot]</code></td><td><code>Boolean</code></td><td><code>true</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If true, it will save a full screenshot of the current page with as a record with key appended by <code>.png</code>.</p>
+</td></tr><tr>
+<td><code>[options.saveHtml]</code></td><td><code>Boolean</code></td><td><code>true</code></td>
+</tr>
+<tr>
+<td colspan="3"><p>If true, it will save a full HTML of the current page with as a record with key appended by <code>.html</code>.</p>
+</td></tr><tr>
+<td><code>[options.storeName]</code></td><td><code>String</code></td><td><code></code></td>
+</tr>
+<tr>
+<td colspan="3"><p>Name or id of the Key-Value store where snapshot is saved. By default it is saved to default Key-Value store.</p>
+</td></tr></tbody>
+</table>
<a name="puppeteer.injectFile"></a>
## `puppeteer.injectFile(page, filePath, [options])` ⇒ `Promise`
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 1164b85c1..e62bb8421 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -11,8 +11,7 @@ import Request from './request';
import { enqueueLinks } from './enqueue_links/enqueue_links';
import { enqueueLinksByClickingElements } from './enqueue_links/click_elements';
import { addInterceptRequestHandler, removeInterceptRequestHandler } from './puppeteer_request_interception';
-import { apifyClient } from './utils';
-import { setValue } from './key_value_store';
+import { openKeyValueStore } from './key_value_store';
const jqueryPath = require.resolve('jquery/dist/jquery.min');
const underscorePath = require.resolve('underscore/underscore-min');
@@ -513,27 +512,52 @@ export const infiniteScroll = async (page, options = {}) => {
}
};
-const saveSnapshot = async (page, key, options = {}) => {
+/**
+ * Saves a full screenshot and HTML of the current page into a Key-Value store.
+ * @param {Object} page
+ * Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
+ * @param {Object} [options]
+ * @param {String} [options.key=SNAPSHOT]
+ * Key under which the screenshot and HTML will be saved. `.png` will be appended for screenshot and `.html` for HTML.
+ * Must contain only letters, numbers, dashes, dots and underscores.
+ * @param {Boolean} [options.saveScreenshot=true]
+ * If true, it will save a full screenshot of the current page as a record with `key` appended by `.png`.
+ * @param {Boolean} [options.saveHtml=true]
+ * If true, it will save a full HTML of the current page as a record with `key` appended by `.html`.
+ * @param {String} [options.storeName=null]
+ * Name or id of the Key-Value store where snapshot is saved. By default it is saved to default Key-Value store.
+ * @returns {Promise}
+ * @memberOf puppeteer
+ * @name saveSnapshot
+ */
+const saveSnapshot = async (page, options = {}) => {
+ const DEFAULT_KEY = 'SNAPSHOT';
+ let key;
try {
checkParamOrThrow(page, 'page', 'Object');
checkParamOrThrow(options, 'options', 'Object');
- const { saveScreenshot = true, saveHtml = true } = options;
+ const { saveScreenshot = true, saveHtml = true, storeName = null } = options;
+ key = options.key || DEFAULT_KEY;
+
+ checkParamOrThrow(saveScreenshot, 'saveScreenshot', 'Boolean');
+ checkParamOrThrow(saveHtml, 'saveHtml', 'Boolean');
+ checkParamOrThrow(key, 'key', 'String');
+ checkParamOrThrow(storeName, 'storeName', 'Maybe String');
- checkParamOrThrow(saveScreenshot, 'timeoutSecs', 'Boolean');
- checkParamOrThrow(saveHtml, 'waitForSecs', 'Boolean');
+ const store = await openKeyValueStore(storeName);
if (saveScreenshot) {
const screenshotBuffer = await page.screenshot({ fullPage: true });
- await setValue(`${key}.png`, screenshotBuffer, { contentType: 'image/png' });
+ await store.setValue(`${key}.png`, screenshotBuffer, { contentType: 'image/png' });
}
if (saveHtml) {
const html = await page.content();
- await setValue(`${key}.html`, html, { contentType: 'text/html' });
+ await store.setValue(`${key}.html`, html, { contentType: 'text/html' });
}
} catch (e) {
// I like this more than having to investigate stack trace
- log.error(`saveSnapshot with key ${key} failed with error:`);
+ log.error(`saveSnapshot with key ${key || ''} failed with error:`);
throw e;
}
};
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index f9af2260e..35d1feca4 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -377,39 +377,40 @@ describe('Apify.utils.puppeteer', () => {
}
});
-
it('saveSnapshot() works', async () => {
const mock = sinon.mock(keyValueStore);
const browser = await Apify.launchPuppeteer({ headless: true });
try {
const page = await browser.newPage();
- let count = 0;
- const content = Array(10).fill(null).map(() => {
- return `<div style="border: 1px solid black">Div number: ${count++}</div>`;
- });
- const contentHTML = `<html><head></head><body>${content}</body></html>`;
+ const contentHTML = '<html><head></head><body><div style="border: 1px solid black">Div number: 1</div></body></html>';
await page.setContent(contentHTML);
const screenshot = await page.screenshot({ fullPage: true });
- mock.expects('setValue')
- .once()
- .withArgs('TEST.png', screenshot, { contentType: 'image/png' })
- .returns(Promise.resolve());
+ // Test saving both image and html
+ const object = { setValue: async () => {} };
+ const stub = sinon.stub(object, 'setValue');
- mock.expects('setValue')
+ mock.expects('openKeyValueStore')
.once()
- .withArgs('TEST.html', contentHTML, { contentType: 'text/html' })
- .returns(Promise.resolve());
+ .withExactArgs('TEST-STORE')
+ .resolves(object);
- await Apify.utils.puppeteer.saveSnapshot(page, 'TEST');
+ await Apify.utils.puppeteer.saveSnapshot(page, { key: 'TEST', storeName: 'TEST-STORE' });
- mock.expects('setValue')
- .once()
- .withArgs('TEST.png', screenshot, { contentType: 'image/png' })
- .returns(Promise.resolve());
+ expect(stub.calledWithExactly('TEST.png', screenshot, { contentType: 'image/png' })).to.be.eql(true);
+ expect(stub.calledWithExactly('TEST.html', contentHTML, { contentType: 'text/html' })).to.be.eql(true);
+
+ // Test saving only image
+ const object2 = { setValue: async () => {} };
+ const stub2 = sinon.stub(object2, 'setValue');
+ mock.expects('openKeyValueStore')
+ .withExactArgs(null)
+ .resolves(object2);
+
+ await Apify.utils.puppeteer.saveSnapshot(page, { saveHtml: false });
- await Apify.utils.puppeteer.saveSnapshot(page, 'TEST', { saveHtml: false });
+ expect(stub2.calledOnceWithExactly('SNAPSHOT.png', screenshot, { contentType: 'image/png' })).to.be.eql(true);
mock.verify();
} finally {