Lightpanda: the headless browser designed for AI and automation
telemetry: Optimize the payload size, add navigation type
Karl Seguin47cffb881993
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zigindex 4f0207366..2d5526506 100644--- a/src/browser/Frame.zig+++ b/src/browser/Frame.zig@@ -652,13 +652,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo.timestamp = timestamp(.monotonic),});- // Record telemetry for navigation- session.browser.app.telemetry.record(.{- .navigate = .{- .tls = false, // about:blank and blob: are not TLS- .proxy = session.browser.app.config.httpProxy() != null,- },- });+ self.recordNavigateTelemetry(false);session.notification.dispatch(.frame_navigated, &.{.req_id = req_id,@@ -738,10 +732,7 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo});// Record telemetry for navigation- session.browser.app.telemetry.record(.{ .navigate = .{- .tls = std.ascii.startsWithIgnoreCase(self.url, "https://"),- .proxy = session.browser.app.config.httpProxy() != null,- } });+ self.recordNavigateTelemetry(std.ascii.startsWithIgnoreCase(self.url, "https://"));session.navigation._current_navigation_kind = opts.kind;@@ -767,6 +758,18 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo};}+fn recordNavigateTelemetry(self: *Frame, tls: bool) void {+ self._session.browser.app.telemetry.record(.{ .navigate = .{+ .tls = tls,+ .context = if (self.parent != null)+ .iframe+ else if (self.window._opener != null)+ .popup+ else+ .page,+ } });+}+// Navigation can happen in many places, such as executing a <script> tag or// a JavaScript callback, a CDP command, etc...It's rarely safe to do immediately// as the caller almost certainly doesn't expect the frame to go away during thediff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zigindex 2ab1eb08f..7caa8a801 100644--- a/src/telemetry/lightpanda.zig+++ b/src/telemetry/lightpanda.zig@@ -19,7 +19,7 @@ const MAX_PENDING = 4096; // hard cap: drop + count beyond this (reported via buconst RECLAIM_CAPACITY = 64; // reclaim the drain buffer once a burst grows it past thisconst MAX_BODY_SIZE = 500 * 1024; // 500KBconst REQUEST_TIMEOUT_MS = 5000;-const URL = "https://telemetry.lightpanda.io";+const URL = "https://telemetry.lightpanda.io/v2";const LightPanda = @This();@@ -33,6 +33,13 @@ iid: ?[36]u8 = null,run_mode: Config.RunMode = .serve,interactive: bool = false,+// Per run id+sid: [16]u8,+proxy: bool,++// Header is sent before the first message, once sent, it isn't sent again+header_sent: bool = false,+// `mutex` guards the ring buffer (head/tail/dropped), `running`, and the lazy// `thread` creation. The sender thread blocks on `cond` while idle.mutex: std.Thread.Mutex = .{},@@ -56,10 +63,15 @@ pending: std.ArrayList(telemetry.Event) = .empty,dropped: u32 = 0,pub fn init(self: *LightPanda, app: *App, iid: ?[36]u8, run_mode: Config.RunMode, interactive: bool) !void {+ var raw_sid: [8]u8 = undefined;+ std.crypto.random.bytes(&raw_sid);+self.* = .{.iid = iid,.run_mode = run_mode,.interactive = interactive,+ .sid = std.fmt.bytesToHex(raw_sid, .lower),+ .proxy = app.config.httpProxy() != null,.allocator = app.allocator,.network = &app.network,.writer = std.Io.Writer.Allocating.init(app.allocator),@@ -170,6 +182,10 @@ fn run(self: *LightPanda) void {}fn postEvents(self: *LightPanda, conn: *http.Connection, events: []const telemetry.Event, dropped: u32, sent: *usize) !void {+ if (self.header_sent == false) {+ _ = try self.writeHeader();+ }+// The overflow report rides ahead of the first body; the rest of that body// and any subsequent ones are filled from `events` below.const has_overflow = dropped > 0;@@ -203,6 +219,8 @@ fn postEvents(self: *LightPanda, conn: *http.Connection, events: []const telemettry self.flush(conn);sent.* += queued;}++ self.header_sent = true;}fn flush(self: *LightPanda, conn: *http.Connection) !void {@@ -223,17 +241,24 @@ fn _flush(self: *LightPanda, conn: *http.Connection) !void {}fn writeEvent(self: *LightPanda, event: telemetry.Event) !bool {+ return self.writeLine(&EventRow{ .sid = &self.sid, .event = event });+}++fn writeHeader(self: *LightPanda) !bool {const iid: ?[]const u8 = if (self.iid) |*id| id else null;- const wrapped = LightPandaEvent{+ return self.writeLine(&Header{+ .sid = &self.sid,.iid = iid,.mode = self.run_mode,- .event = event,.interactive = self.interactive,- };+ .proxy = self.proxy,+ });+}+fn writeLine(self: *LightPanda, value: anytype) !bool {const checkpoint = self.writer.written().len;- try std.json.Stringify.value(&wrapped, .{ .emit_null_optional_fields = false }, &self.writer.writer);+ try std.json.Stringify.value(value, .{}, &self.writer.writer);try self.writer.writer.writeByte('\n');if (self.writer.written().len > MAX_BODY_SIZE) {@@ -243,17 +268,23 @@ fn writeEvent(self: *LightPanda, event: telemetry.Event) !bool {return true;}-const LightPandaEvent = struct {+const Header = struct {+ sid: []const u8,iid: ?[]const u8,mode: Config.RunMode,interactive: bool,- event: telemetry.Event,+ proxy: bool,- pub fn jsonStringify(self: *const LightPandaEvent, writer: anytype) !void {+ pub fn jsonStringify(self: *const Header, writer: anytype) !void {try writer.beginObject();- try writer.objectField("iid");- try writer.write(self.iid);+ try writer.objectField("sid");+ try writer.write(self.sid);++ if (self.iid) |iid| {+ try writer.objectField("iid");+ try writer.write(iid);+ }try writer.objectField("mode");// Special case: when running agent mode in non-interactive, we send a@@ -273,22 +304,95 @@ const LightPandaEvent = struct {try writer.objectField("version");try writer.write(build_config.version);- try writer.objectField("event");- try writer.write(@tagName(std.meta.activeTag(self.event)));-- inline for (@typeInfo(telemetry.Event).@"union".fields) |union_field| {- if (self.event == @field(telemetry.Event, union_field.name)) {- const inner = @field(self.event, union_field.name);- const TI = @typeInfo(@TypeOf(inner));- if (TI == .@"struct") {- inline for (TI.@"struct".fields) |field| {- try writer.objectField(field.name);- try writer.write(@field(inner, field.name));- }- }- }- }+ try writer.objectField("proxy");+ try writer.write(self.proxy);try writer.endObject();}};++const EventRow = struct {+ sid: []const u8,+ event: telemetry.Event,++ pub fn jsonStringify(self: *const EventRow, writer: anytype) !void {+ try writer.beginArray();+ try writer.write(self.sid);+ switch (self.event) {+ .run => try writer.write("run"),+ .navigate => |n| {+ try writer.write("nav");+ try writer.write(n.tls);+ try writer.write(n.context);+ },+ .buffer_overflow => |b| {+ try writer.write("bof");+ try writer.write(b.dropped);+ },+ .llm => |l| {+ try writer.write("llm");+ try writer.write(l.provider);+ try writer.write(l.model);+ },+ }+ try writer.endArray();+ }+};++const testing = @import("../testing.zig");++test "telemetry: event row wire format" {+ const Case = struct { event: telemetry.Event, expected: []const u8 };+ const cases = [_]Case{+ .{ .event = .{ .run = {} }, .expected = "[\"sid0\",\"run\"]" },+ .{+ .event = .{ .navigate = .{ .tls = true, .context = .page, .reason = .address_bar } },+ .expected = "[\"sid0\",\"navigate\",true,\"page\",\"address_bar\"]",+ },+ .{+ .event = .{ .navigate = .{ .tls = false, .context = .popup, .reason = .anchor } },+ .expected = "[\"sid0\",\"navigate\",false,\"popup\",\"anchor\"]",+ },+ .{+ .event = .{ .navigate = .{ .tls = true, .context = .iframe, .reason = .script } },+ .expected = "[\"sid0\",\"navigate\",true,\"iframe\",\"script\"]",+ },+ .{+ .event = .{ .buffer_overflow = .{ .dropped = 42 } },+ .expected = "[\"sid0\",\"buffer_overflow\",42]",+ },+ .{+ .event = .{ .llm = .{ .provider = "anthropic", .model = .wrap("claude") } },+ .expected = "[\"sid0\",\"llm\",\"anthropic\",\"claude\"]",+ },+ .{+ .event = .{ .llm = .{ .provider = "nollm", .model = null } },+ .expected = "[\"sid0\",\"llm\",\"nollm\",null]",+ },+ };++ for (cases) |case| {+ var w = std.Io.Writer.Allocating.init(testing.allocator);+ defer w.deinit();+ try std.json.Stringify.value(&EventRow{ .sid = "sid0", .event = case.event }, .{}, &w.writer);+ try testing.expectEqual(case.expected, w.written());+ }+}++test "telemetry: header wire format" {+ var w = std.Io.Writer.Allocating.init(testing.allocator);+ defer w.deinit();++ const header = Header{+ .sid = "sid0",+ .iid = "the-iid",+ .mode = .serve,+ .interactive = false,+ .proxy = true,+ };+ try std.json.Stringify.value(&header, .{}, &w.writer);++ const out = w.written();+ try testing.expectEqual(true, std.mem.startsWith(u8, out, "{\"sid\":\"sid0\",\"iid\":\"the-iid\",\"mode\":\"serve\","));+ try testing.expectEqual(true, std.mem.endsWith(u8, out, ",\"proxy\":true,\"driver\":\"cdp\"}"));+}diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zigindex dfd88462c..f8436c9b1 100644--- a/src/telemetry/telemetry.zig+++ b/src/telemetry/telemetry.zig@@ -111,8 +111,9 @@ pub const Event = union(enum) {const Navigate = struct {tls: bool,- proxy: bool,- driver: enum { cdp } = .cdp,+ context: Context,++ const Context = enum { page, iframe, popup };};const BufferOverflow = struct {