Skip to content

Lightpanda: the headless browser designed for AI and automation

fetch: add --dump structured_data

Adrià Arrufat3ced7057150d
diff --git a/README.md b/README.md
index 43842a232..d2bdf4a59 100644
--- a/README.md
+++ b/README.md
@@ -110,7 +110,8 @@ docker run -d --name lightpanda -p 127.0.0.1:9222:9222 lightpanda/browser:latest
You can use `--dump markdown` to convert directly into markdown, or
`--dump png > page.png` or `--dump pdf > page.pdf` for a text-only rendering
-of the page.
+of the page. `--dump structured_data` prints the page's JSON-LD, OpenGraph,
+Twitter card, meta and link tags as JSON, like the MCP `structuredData` tool.
`--wait-until`, `--wait-ms`, `--wait-selector` and `--wait-script` are
available to adjust waiting time before dump.
diff --git a/src/Config.zig b/src/Config.zig
index eab3dd4a4..71ed02546 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -1037,6 +1037,7 @@ pub const DumpFormat = enum {
wpt,
semantic_tree,
semantic_tree_text,
+ structured_data,
};
pub const Shell = enum { bash, fish, zsh };
diff --git a/src/browser/tests/structured_data.html b/src/browser/tests/structured_data.html
new file mode 100644
index 000000000..7e9d3080c
--- /dev/null
+++ b/src/browser/tests/structured_data.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <title>Blue Kettle</title>
+ <meta property="og:type" content="product">
+ <link rel="canonical" href="/products/blue-kettle">
+ <script type="application/ld+json">{"@type":"Product","name":"Blue Kettle"}</script>
+</head>
+<body><p>A kettle.</p></body>
+</html>
diff --git a/src/help.zon b/src/help.zon
index fa7cf23d1..7008da4f9 100644
--- a/src/help.zon
+++ b/src/help.zon
@@ -103,6 +103,8 @@
\\ PNG image (base64 with --json).
\\ semantic_tree JSON-serialized semantic tree.
\\ semantic_tree_text Pruned plain-text semantic tree.
+ \\ structured_data JSON-LD, OpenGraph, Twitter card, meta
+ \\ and link tags as JSON.
\\ --dump-max-bytes <INT>
\\ Soft cap on the dumped html or markdown, in bytes. Output is cut
\\ at a UTF-8 boundary and a '[truncated]' marker is appended.
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index a86eb6621..d27173354 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -523,6 +523,10 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Fram
try st.textStringify(writer);
}
},
+ .structured_data => {
+ const data = try structured_data.collectStructuredData(state.root, arena.allocator(), frame);
+ try std.json.Stringify.value(data, .{}, writer);
+ },
.wpt => try dumpWPT(frame, writer),
}
}
@@ -700,6 +704,20 @@ test "writeJsonEnvelope: null frame with dump mode and content" {
}, aw.written());
}
+test "dumpContent: structured_data" {
+ var page = try testing.pageTest("structured_data.html", .{});
+ defer page.close();
+
+ var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
+ try dumpContent(testing.test_app, .structured_data, .{ .dump = .{} }, page.frame().?, &aw.writer);
+ try testing.expectJson(.{
+ .jsonLd = &.{"{\"@type\":\"Product\",\"name\":\"Blue Kettle\"}"},
+ .openGraph = .{ .type = "product" },
+ .meta = .{ .language = "en", .title = "Blue Kettle" },
+ .links = .{ .canonical = "http://127.0.0.1:9582/products/blue-kettle" },
+ }, aw.written());
+}
+
test "fetch: resolveWaitUntil" {
try testing.expectEqual(.load, resolveWaitUntil(.{ .dump = .{} }));
try testing.expectEqual(.done, resolveWaitUntil(.{ .dump = .{}, .wait_until = .done }));