Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
Add screenshot and PDF fetch dumps
ldm0bc7379670233
diff --git a/moli-protocol/src/domains/page.rs b/moli-protocol/src/domains/page.rsindex 628b65002..509e147a3 100644--- a/moli-protocol/src/domains/page.rs+++ b/moli-protocol/src/domains/page.rs@@ -65,6 +65,22 @@ mod termination;#[cfg(test)]mod tests;+/// Builds a letter-sized raster PDF using the same defaults as+/// `Page.printToPDF`.+pub fn build_default_raster_pdf(+ jpeg: &[u8],+ image_width: u32,+ image_height: u32,+) -> anyhow::Result<Vec<u8>> {+ pdf::build_raster_pdf(+ jpeg,+ image_width,+ image_height,+ &pdf::RasterPdfOptions::default(),+ )+ .map_err(|error| anyhow::anyhow!(error.message().to_owned()))+}+use child_frame_activity::PagePreparedChildFrameDocumentActivity;pub(crate) use child_frame_activity::{PagePreparedChildFrameActivity, PagePreparedChildFrameTreeEvent,diff --git a/moli-protocol/src/domains/page/pdf.rs b/moli-protocol/src/domains/page/pdf.rsindex 5d86a7059..be264e3e5 100644--- a/moli-protocol/src/domains/page/pdf.rs+++ b/moli-protocol/src/domains/page/pdf.rs@@ -21,6 +21,21 @@ pub(super) struct RasterPdfOptions {page_ranges: Vec<PageRange>,}+impl Default for RasterPdfOptions {+ fn default() -> Self {+ Self {+ page_width_points: DEFAULT_PAGE_WIDTH_INCHES * POINTS_PER_INCH,+ page_height_points: DEFAULT_PAGE_HEIGHT_INCHES * POINTS_PER_INCH,+ margin_top_points: DEFAULT_MARGIN_INCHES * POINTS_PER_INCH,+ margin_bottom_points: DEFAULT_MARGIN_INCHES * POINTS_PER_INCH,+ margin_left_points: DEFAULT_MARGIN_INCHES * POINTS_PER_INCH,+ margin_right_points: DEFAULT_MARGIN_INCHES * POINTS_PER_INCH,+ scale: 1.0,+ page_ranges: Vec::new(),+ }+ }+}+#[derive(Clone, Copy, Debug, PartialEq, Eq)]struct PageRange {first: usize,diff --git a/moli-protocol/src/lib.rs b/moli-protocol/src/lib.rsindex d0188eb0a..2b92c36f2 100644--- a/moli-protocol/src/lib.rs+++ b/moli-protocol/src/lib.rs@@ -39,7 +39,7 @@ pub use domains::activity::{pub use domains::page::{BackgroundNavigationCompletion, CompletedPageScreencastCapture,PageScreencastCaptureCompletion, PageScreencastCaptureStart, PageScreencastRegistration,- PageScreencastSubscriptionStatus, PendingPageScreencastCapture,+ PageScreencastSubscriptionStatus, PendingPageScreencastCapture, build_default_raster_pdf,};pub use domains::runtime::{CompletedDevToolsRuntimeCommandDispatch, DevToolsRuntimeCommandTaskStep,diff --git a/moli/src/app.rs b/moli/src/app.rsindex df579ba55..d1d64eb8a 100644--- a/moli/src/app.rs+++ b/moli/src/app.rs@@ -181,11 +181,11 @@ pub async fn run_cli_with_config<W: Write>(.context("failed while waiting for page delay")?;}- let rendered = fetch_dump::render_page_dump_async(&mut page, &config.fetch)+ let rendered = fetch_dump::render_page_output_async(&mut page, &config.fetch).await.context("failed to render fetch output")?;stdout- .write_all(rendered.as_bytes())+ .write_all(&rendered).context("failed to write fetch output")?;let _ = stdout.flush();if let Err(error) = page.close_async().await {diff --git a/moli/src/cli.rs b/moli/src/cli.rsindex 0e71d7c70..5342e403e 100644--- a/moli/src/cli.rs+++ b/moli/src/cli.rs@@ -39,6 +39,8 @@ const DUMP_MODES: &[&str] = &["json","html","markdown",+ "screenshot",+ "pdf","wpt","semantic_tree","semantic_tree_text",@@ -68,6 +70,8 @@ pub enum Commands {#[derive(Debug, Clone, PartialEq, Eq, Args)]pub struct FetchArgs {+ /// Select the fetch output format. `screenshot` writes a viewport PNG and+ /// `pdf` writes a paginated PDF directly to stdout; both require layout.#[arg(long, value_enum)]pub dump: Option<DumpFormat>,@@ -422,6 +426,8 @@ pub enum DumpFormat {Json,Html,Markdown,+ Screenshot,+ Pdf,Wpt,SemanticTree,SemanticTreeText,diff --git a/moli/src/fetch_dump.rs b/moli/src/fetch_dump.rsindex 4a84293ab..65fb35006 100644--- a/moli/src/fetch_dump.rs+++ b/moli/src/fetch_dump.rs@@ -1,7 +1,9 @@-use anyhow::Result;+use anyhow::{Context, Result, bail};use moli_core::page::{- DocumentNodeSnapshot, Page, RendererPageDumpFormat, RendererPageDumpOptions,- RendererPageDumpStripOptions, SubresourceResponseWaitCriteria, is_renderer_backend_node_id,+ DocumentNodeSnapshot, Page, RendererCaptureScreenshotReply, RendererCaptureScreenshotRequest,+ RendererPageDumpFormat, RendererPageDumpOptions, RendererPageDumpStripOptions,+ RendererScreenshotFormat, RendererScreenshotPurpose, RendererScreenshotRegion,+ SubresourceResponseWaitCriteria, is_renderer_backend_node_id,};use moli_core::runtime::RawDocument;#[cfg(test)]@@ -95,6 +97,26 @@ const WPT_DUMP_SCRIPT: &str = r#"})()"#;+struct CapturedRaster {+ mime_type: String,+ width: u32,+ height: u32,+ bytes: Vec<u8>,+}++pub async fn render_page_output_async(+ page: &mut Page,+ command: &FetchCommandConfig,+) -> Result<Vec<u8>> {+ match command.dump_mode.unwrap_or(DumpFormat::Html) {+ DumpFormat::Screenshot => render_screenshot_dump_async(page).await,+ DumpFormat::Pdf => render_pdf_dump_async(page).await,+ _ => render_page_dump_async(page, command)+ .await+ .map(String::into_bytes),+ }+}+pub async fn render_page_dump_async(page: &mut Page,command: &FetchCommandConfig,@@ -128,6 +150,8 @@ pub fn render_raw_document_dump(Ok(payload.into_bytes())}DumpFormat::Markdown+ | DumpFormat::Screenshot+ | DumpFormat::Pdf| DumpFormat::Wpt| DumpFormat::SemanticTree| DumpFormat::SemanticTreeText => {@@ -186,12 +210,86 @@ async fn render_page_dump_with_trace_config_async(}DumpFormat::Html => render_html_dump_async(page, strip, with_base, with_frames).await,DumpFormat::Markdown => render_markdown_dump_async(page, strip, with_frames).await,+ DumpFormat::Screenshot | DumpFormat::Pdf => {+ bail!("binary dump formats are only supported by the fetch CLI output path")+ }DumpFormat::SemanticTree => render_semantic_tree_dump_async(page).await,DumpFormat::SemanticTreeText => render_semantic_tree_text_dump_async(page).await,DumpFormat::Wpt => render_wpt_dump_async(page).await,}}+async fn render_screenshot_dump_async(page: &mut Page) -> Result<Vec<u8>> {+ let image = capture_page_raster(+ page,+ RendererCaptureScreenshotRequest::viewport_png(),+ "screenshot",+ )+ .await?;+ if image.mime_type != "image/png" {+ bail!(+ "screenshot renderer returned `{}` instead of PNG",+ image.mime_type+ );+ }+ Ok(image.bytes)+}++async fn render_pdf_dump_async(page: &mut Page) -> Result<Vec<u8>> {+ let image = capture_page_raster(+ page,+ RendererCaptureScreenshotRequest {+ purpose: RendererScreenshotPurpose::Print {+ print_background: false,+ },+ format: RendererScreenshotFormat::Jpeg,+ quality: 90,+ region: RendererScreenshotRegion::FullDocument,+ optimize_for_speed: false,+ max_width: None,+ max_height: None,+ },+ "pdf",+ )+ .await?;+ if image.mime_type != "image/jpeg" {+ bail!(+ "PDF renderer returned `{}` instead of JPEG",+ image.mime_type+ );+ }+ moli_protocol::build_default_raster_pdf(&image.bytes, image.width, image.height)+ .context("failed to encode PDF output")+}++async fn capture_page_raster(+ page: &mut Page,+ request: RendererCaptureScreenshotRequest,+ dump_mode: &str,+) -> Result<CapturedRaster> {+ let pending = page+ .start_capture_screenshot_with_request(request)+ .with_context(|| format!("failed to start --dump {dump_mode} capture"))?;+ let completion = pending+ .wait()+ .await+ .with_context(|| format!("failed to complete --dump {dump_mode} capture"))?;+ match page.finish_capture_screenshot(completion)? {+ RendererCaptureScreenshotReply::Captured(image) => Ok(CapturedRaster {+ mime_type: image.mime_type,+ width: image.width,+ height: image.height,+ bytes: image.bytes.to_vec(),+ }),+ RendererCaptureScreenshotReply::LayoutDisabled => {+ bail!("--dump {dump_mode} requires --layout or MOLI_LAYOUT=true")+ }+ RendererCaptureScreenshotReply::NoDocument => {+ bail!("--dump {dump_mode} requires a loaded HTML document")+ }+ }+}+async fn render_json_dump_async(page: &mut Page,strip: StripOptions,diff --git a/moli/tests/cli.rs b/moli/tests/cli.rsindex 1f20a6a1e..25c9bbedd 100644--- a/moli/tests/cli.rs+++ b/moli/tests/cli.rs@@ -174,6 +174,27 @@ fn parses_json_dump_mode() {assert_eq!(args.dump, Some(DumpFormat::Json));}+#[test]+fn parses_binary_dump_modes_with_inferred_fetch_command() {+ for (value, expected) in [+ ("screenshot", DumpFormat::Screenshot),+ ("pdf", DumpFormat::Pdf),+ ] {+ let cli = Cli::try_parse_from(normalize_args_for_compat([+ "moli",+ "--dump",+ value,+ "https://example.com",+ ]))+ .unwrap();++ let Commands::Fetch(args) = cli.command else {+ panic!("expected fetch command for --dump {value}");+ };+ assert_eq!(args.dump, Some(expected));+ }+}+#[test]fn app_config_rejects_unimplemented_web_bot_auth_flags() {let cli = Cli::try_parse_from(normalize_args_for_compat([diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex fda2ca95c..2408b007b 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -774,6 +774,61 @@ fn cli_fetch_about_blank_materializes_empty_document() -> Result<()> {Ok(())}+#[test]+fn cli_dump_screenshot_writes_png_bytes() -> Result<()> {+ let output = run_fetch_cli_with_dump_and_args("about:blank", "screenshot", &["--layout"])?;+ assert!(+ output.status.success(),+ "moli fetch failed: stdout_bytes={}\nstderr={}",+ output.stdout.len(),+ String::from_utf8_lossy(&output.stderr)+ );+ assert!(output.stdout.starts_with(b"\x89PNG\r\n\x1a\n"));+ assert!(output.stdout.ends_with(b"IEND\xaeB`\x82"));+ Ok(())+}++#[test]+fn cli_dump_pdf_writes_pdf_bytes() -> Result<()> {+ let output = run_fetch_cli_with_dump_and_args("about:blank", "pdf", &["--layout"])?;+ assert!(+ output.status.success(),+ "moli fetch failed: stdout_bytes={}\nstderr={}",+ output.stdout.len(),+ String::from_utf8_lossy(&output.stderr)+ );+ assert!(output.stdout.starts_with(b"%PDF-1.7\n"));+ assert!(+ output+ .stdout+ .windows(b"/Type /Pages".len())+ .any(|window| window == b"/Type /Pages")+ );+ assert!(output.stdout.ends_with(b"%%EOF\n"));+ Ok(())+}++#[test]+fn binary_dump_modes_require_layout() -> Result<()> {+ for dump in ["screenshot", "pdf"] {+ let output = run_fetch_cli_with_dump_and_args("about:blank", dump, &[])?;+ assert!(+ !output.status.success(),+ "--dump {dump} unexpectedly worked"+ );+ assert!(+ output.stdout.is_empty(),+ "--dump {dump} wrote partial output"+ );+ let stderr = String::from_utf8_lossy(&output.stderr);+ assert!(+ stderr.contains(&format!("--dump {dump} requires --layout")),+ "stderr={stderr}"+ );+ }+ Ok(())+}+#[test]fn cli_fetch_inline_pdf_dump_html_bypasses_dcl_page_lifecycle() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;