Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: add `utils.playwright.saveSnapshot` (#1544)
Jindřich Bära4ceef044f0c
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex dad4b0f9d..b1bff55c2 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -24,7 +24,7 @@ import type { Page, Response, Route } from 'playwright';import { LruCache } from '@apify/datastructures';import log_ from '@apify/log';import type { Request } from '@crawlee/browser';-import { validators } from '@crawlee/browser';+import { validators, KeyValueStore } from '@crawlee/browser';import type { CheerioRoot, Dictionary } from '@crawlee/utils';import * as cheerio from 'cheerio';import type { PlaywrightCrawlingContext } from '../playwright-crawler';@@ -406,6 +406,80 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptions}}+export interface SaveSnapshotOptions {+ /**+ * Key under which the screenshot and HTML will be saved. `.jpg` will be appended for screenshot and `.html` for HTML.+ * @default 'SNAPSHOT'+ */+ key?: string;++ /**+ * The quality of the image, between 0-100. Higher quality images have bigger size and require more storage.+ * @default 50+ */+ screenshotQuality?: number;++ /**+ * If true, it will save a full screenshot of the current page as a record with `key` appended by `.jpg`.+ * @default true+ */+ saveScreenshot?: boolean;++ /**+ * If true, it will save a full HTML of the current page as a record with `key` appended by `.html`.+ * @default true+ */+ saveHtml?: boolean;++ /**+ * Name or id of the Key-Value store where snapshot is saved. By default it is saved to default Key-Value store.+ * @default null+ */+ keyValueStoreName?: string | null;+}++/**+ * Saves a full screenshot and HTML of the current page into a Key-Value store.+ * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object.+ * @param [options]+ */+export async function saveSnapshot(page: Page, options: SaveSnapshotOptions = {}): Promise<void> {+ ow(page, ow.object.validate(validators.browserPage));+ ow(options, ow.object.exactShape({+ key: ow.optional.string.nonEmpty,+ screenshotQuality: ow.optional.number,+ saveScreenshot: ow.optional.boolean,+ saveHtml: ow.optional.boolean,+ keyValueStoreName: ow.optional.string,+ }));++ const {+ key = 'SNAPSHOT',+ screenshotQuality = 50,+ saveScreenshot = true,+ saveHtml = true,+ keyValueStoreName,+ } = options;++ try {+ const store = await KeyValueStore.open(keyValueStoreName);++ if (saveScreenshot) {+ const screenshotName = `${key}.jpg`;+ const screenshotBuffer = await page.screenshot({ fullPage: true, quality: screenshotQuality, type: 'jpeg' });+ await store.setValue(screenshotName, screenshotBuffer, { contentType: 'image/jpeg' });+ }++ if (saveHtml) {+ const htmlName = `${key}.html`;+ const html = await page.content();+ await store.setValue(htmlName, html, { contentType: 'text/html' });+ }+ } catch (err) {+ throw new Error(`saveSnapshot with key ${key} failed.\nCause:${(err as Error).message}`);+ }+}+/*** Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@apilink CheerioCrawler}.*@@ -429,6 +503,7 @@ export interface PlaywrightContextUtils {blockRequests(options?: BlockRequestsOptions): Promise<void>;parseWithCheerio(): Promise<CheerioRoot>;infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;+ saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;}export function registerUtilsToContext(context: PlaywrightCrawlingContext): void {@@ -437,6 +512,7 @@ export function registerUtilsToContext(context: PlaywrightCrawlingContext): voidcontext.blockRequests = (options?: BlockRequestsOptions) => blockRequests(context.page, options);context.parseWithCheerio = () => parseWithCheerio(context.page);context.infiniteScroll = (options?: InfiniteScrollOptions) => infiniteScroll(context.page, options);+ context.saveSnapshot = (options?: SaveSnapshotOptions) => saveSnapshot(context.page, options);}/** @internal */@@ -447,4 +523,5 @@ export const playwrightUtils = {blockRequests,parseWithCheerio,infiniteScroll,+ saveSnapshot,};diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.tsindex b86b3e87a..c7ddf6ed7 100644--- a/test/core/playwright_utils.test.ts+++ b/test/core/playwright_utils.test.ts@@ -3,7 +3,7 @@ import path from 'path';import log from '@apify/log';import type { Server } from 'http';import type { AddressInfo } from 'net';-import { Request, launchPlaywright, playwrightUtils } from '@crawlee/playwright';+import { KeyValueStore, Request, launchPlaywright, playwrightUtils } from '@crawlee/playwright';import type { Browser, Page } from 'playwright';import { chromium } from 'playwright';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';@@ -298,5 +298,37 @@ describe('playwrightUtils', () => {expect(after).toBe(true);});});++ test('saveSnapshot() works', async () => {+ const openKVSSpy = jest.spyOn(KeyValueStore, 'open');+ const browser = await chromium.launch({ headless: true });++ try {+ const page = await browser.newPage();+ const contentHTML = '<html><head></head><body><div style="border: 1px solid black">Div number: 1</div></body></html>';+ await page.setContent(contentHTML);++ const screenshot = await page.screenshot({ fullPage: true, type: 'jpeg', quality: 60 });++ // Test saving both image and html+ const object = { setValue: jest.fn() };+ openKVSSpy.mockResolvedValue(object as any);+ await playwrightUtils.saveSnapshot(page, { key: 'TEST', keyValueStoreName: 'TEST-STORE', screenshotQuality: 60 });++ expect(object.setValue).toBeCalledWith('TEST.jpg', screenshot, { contentType: 'image/jpeg' });+ expect(object.setValue).toBeCalledWith('TEST.html', contentHTML, { contentType: 'text/html' });+ object.setValue.mockReset();++ // Test saving only image+ await playwrightUtils.saveSnapshot(page, { saveHtml: false });++ // Default quality is 50+ const screenshot2 = await page.screenshot({ fullPage: true, type: 'jpeg', quality: 50 });+ expect(object.setValue).toBeCalledWith('SNAPSHOT.jpg', screenshot2, { contentType: 'image/jpeg' });+ } finally {+ openKVSSpy.mockRestore();+ await browser.close();+ }+ });});});