Lightpanda: the headless browser designed for AI and automation
fetch: add --dump structured_data
Adrià Arrufat3ced7057150d
diff --git a/README.md b/README.mdindex 43842a232..d2bdf4a59 100644--- a/README.md+++ b/README.md@@ -110,7 +110,8 @@ docker run -d --name lightpanda -p 127.0.0.1:9222:9222 lightpanda/browser:latestYou can use `--dump markdown` to convert directly into markdown, or`--dump png > page.png` or `--dump pdf > page.pdf` for a text-only rendering-of the page.+of the page. `--dump structured_data` prints the page's JSON-LD, OpenGraph,+Twitter card, meta and link tags as JSON, like the MCP `structuredData` tool.`--wait-until`, `--wait-ms`, `--wait-selector` and `--wait-script` areavailable to adjust waiting time before dump.diff --git a/src/Config.zig b/src/Config.zigindex eab3dd4a4..71ed02546 100644--- a/src/Config.zig+++ b/src/Config.zig@@ -1037,6 +1037,7 @@ pub const DumpFormat = enum {wpt,semantic_tree,semantic_tree_text,+ structured_data,};pub const Shell = enum { bash, fish, zsh };diff --git a/src/browser/tests/structured_data.html b/src/browser/tests/structured_data.htmlnew file mode 100644index 000000000..7e9d3080c--- /dev/null+++ b/src/browser/tests/structured_data.html@@ -0,0 +1,10 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <title>Blue Kettle</title>+ <meta property="og:type" content="product">+ <link rel="canonical" href="/products/blue-kettle">+ <script type="application/ld+json">{"@type":"Product","name":"Blue Kettle"}</script>+</head>+<body><p>A kettle.</p></body>+</html>diff --git a/src/help.zon b/src/help.zonindex fa7cf23d1..7008da4f9 100644--- a/src/help.zon+++ b/src/help.zon@@ -103,6 +103,8 @@\\ PNG image (base64 with --json).\\ semantic_tree JSON-serialized semantic tree.\\ semantic_tree_text Pruned plain-text semantic tree.+ \\ structured_data JSON-LD, OpenGraph, Twitter card, meta+ \\ and link tags as JSON.\\ --dump-max-bytes <INT>\\ Soft cap on the dumped html or markdown, in bytes. Output is cut\\ at a UTF-8 boundary and a '[truncated]' marker is appended.diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex a86eb6621..d27173354 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -523,6 +523,10 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Framtry st.textStringify(writer);}},+ .structured_data => {+ const data = try structured_data.collectStructuredData(state.root, arena.allocator(), frame);+ try std.json.Stringify.value(data, .{}, writer);+ },.wpt => try dumpWPT(frame, writer),}}@@ -700,6 +704,20 @@ test "writeJsonEnvelope: null frame with dump mode and content" {}, aw.written());}+test "dumpContent: structured_data" {+ var page = try testing.pageTest("structured_data.html", .{});+ defer page.close();++ var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);+ try dumpContent(testing.test_app, .structured_data, .{ .dump = .{} }, page.frame().?, &aw.writer);+ try testing.expectJson(.{+ .jsonLd = &.{"{\"@type\":\"Product\",\"name\":\"Blue Kettle\"}"},+ .openGraph = .{ .type = "product" },+ .meta = .{ .language = "en", .title = "Blue Kettle" },+ .links = .{ .canonical = "http://127.0.0.1:9582/products/blue-kettle" },+ }, aw.written());+}+test "fetch: resolveWaitUntil" {try testing.expectEqual(.load, resolveWaitUntil(.{ .dump = .{} }));try testing.expectEqual(.done, resolveWaitUntil(.{ .dump = .{}, .wait_until = .done }));