Lightpanda: the headless browser designed for AI and automation
add http metrics
Karl Seguinf0784de01560
diff --git a/src/Metrics.zig b/src/Metrics.zigindex 096438fa9..e65148ee7 100644--- a/src/Metrics.zig+++ b/src/Metrics.zig@@ -41,6 +41,36 @@ js_heap_size_bytes: Histogram(&.{256 * 1024 * 1024,512 * 1024 * 1024,}) = .{},+http_requests: CounterEnum("mode", enum { sync, async }) = .{},+http_status: CounterEnum("category", @import("network/http.zig").StatusCategory) = .{},+http_error: CounterEnum("reason", @import("network/http.zig").ErrorReason) = .{},+http_cache: CounterEnum("result", enum { hit, miss, revalidated }) = .{},+http_redirects: Counter = .{},+http_duration_ms: Histogram(&.{+ 5,+ 10,+ 25,+ 50,+ 100,+ 250,+ 500,+ 1000,+ 2500,+ 5000,+ 10000,+}) = .{},+http_response_size_bytes: Histogram(&.{+ 32 * 1024,+ 64 * 1024,+ 128 * 1024,+ 256 * 1024,+ 512 * 1024,+ 1024 * 1024,+ 2 * 1024 * 1024,+ 4 * 1024 * 1024,+}) = .{},+robots_status: CounterEnum("category", @import("network/http.zig").StatusCategory) = .{},+robots_access: CounterEnum("result", enum { allow, deny }) = .{},// Emitted as each metric's "# HELP" line. A field without an entry is a// compile error.@@ -56,6 +86,15 @@ const help = .{.arena_miss = "Arena pool acquisitions that had to allocate a new arena",.navigate = "Navigations by initiating frame type",.js_heap_size_bytes = "V8 heap physical size, sampled when a page is closed",+ .http_requests = "HTTP requests submitted, by dispatch mode (excludes internal requests like robots.txt)",+ .http_status = "Final HTTP response status category (redirects counted once, at the final hop)",+ .http_error = "HTTP requests that failed before delivering a response, by cause",+ .http_cache = "HTTP cache lookups by outcome",+ .http_redirects = "HTTP redirect hops followed",+ .http_duration_ms = "HTTP request wall-clock duration in milliseconds",+ .http_response_size_bytes = "HTTP response body size in bytes",+ .robots_status = "robots.txt response status",+ .robots_access = "robots.txt result",};pub fn write(self: *const Metrics, writer: *std.Io.Writer) void {diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex a8e2aa221..2220d6b94 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -780,8 +780,13 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {}if (self.obey_robots and !transfer.req.internal) {switch (try self.robots.check(transfer)) {- .allowed => {},- .blocked => return transfer.failAsync(error.RobotsBlocked),+ .allowed => {+ lp.metrics.robots_access.incr(.allow);+ },+ .blocked => {+ lp.metrics.robots_access.incr(.deny);+ return transfer.failAsync(error.RobotsBlocked);+ },.pending => return,}}@@ -832,17 +837,20 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {.timestamp = std.time.timestamp(),.request_headers = req_headers.items,}) orelse {+ lp.metrics.http_cache.incr(.miss);transfer._cache_intent = .store;return false;};if (cached.expired == false) {+ lp.metrics.http_cache.incr(.hit);try transfer.bufferCached(cached);return true;}if (cached.metadata.hasValidators() == false) {// Expired and no validators+ lp.metrics.http_cache.incr(.miss);cached.data.deinit();cache.evict(req.url);transfer._cache_intent = .store;@@ -889,6 +897,7 @@ fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool {log.warn(.cache, "renew failed", .{ .err = err });};+ lp.metrics.http_cache.incr(.revalidated);try transfer.bufferCached(stale);return true;}@@ -1021,6 +1030,7 @@ pub fn syncRequest(self: *Client, allocator: Allocator, req: Request) !SyncRespoerrdefer sync_ctx.body.deinit(allocator);var r = req;+ r.sync = true;r.ctx = &sync_ctx;r.header_callback = SyncContext.headerCallback;r.data_callback = SyncContext.dataCallback;@@ -1350,6 +1360,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Tif (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {.follow => {try transfer.handleRedirect(location.value);+ if (!transfer.req.internal) lp.metrics.http_redirects.incr();const conn = transfer._conn.?;@@ -1412,6 +1423,14 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Ttry transfer.materializeResponse(msg.conn);+ // Latency is only meaningful for responses that hit the network (cache+ // and synthetic responses never reach processOneMessage).+ if (!transfer.req.internal) {+ if (msg.conn.getTotalTimeMicros()) |micros| {+ lp.metrics.http_duration_ms.observe(@intCast(@max(0, @divTrunc(micros, 1000))));+ } else |_| {}+ }+// Release the conn before any of this response's callbacks can run —// they'll want it for the next resource.self.removeConn(msg.conn);@@ -1549,6 +1568,9 @@ pub const Request = struct {// these do not need to be deferred and do not obey robots.txt.internal: bool = false,+ // Set by syncRequest; only used to label the http_requests metric.+ sync: bool = false,+// When false, the caller does not guarantee that the body outlives the// transfer, and thus we'll need to dupe it.body_outlives_request: bool = false,@@ -1926,6 +1948,9 @@ pub const Transfer = struct {// on failure the transfer is already cleaned up (error_callback fired,// memory freed) — do not deinit it, and do not touch it after this.pub fn submit(self: *Transfer) anyerror!void {+ if (!self.req.internal) {+ lp.metrics.http_requests.incr(if (self.req.sync) .sync else .async);+ }// Synthetic schemes never touch the network — no robots, cache, or// interception. The response is materialized here and delivered by// the dispatcher like any other transfer.@@ -2136,6 +2161,10 @@ pub const Transfer = struct {}self._notified_fail = true;+ if (!self.req.internal) {+ lp.metrics.http_error.incr(http.errorReason(err));+ }+if (self._notify_cdp) {self.req.notification.dispatch(.http_request_fail, &.{.transfer = self,@@ -2184,6 +2213,17 @@ pub const Transfer = struct {// Buffer the standard success event sequence. `body` is either owned by// transfer.arena OR, through some other mechanism, outlives the transfer.fn bufferEvents(self: *Transfer, body: []const u8) !void {+ // Single delivery point for every successful response — network,+ // cache, and synthetic, on both the sync and async paths. Redirect+ // hops never reach here (they loop back in processOneMessage), so+ // http_status reflects the final response only. Internal requests+ // (robots.txt) are tracked by the robots_* metrics instead.+ if (!self.req.internal) {+ const status = if (self.res.header) |h| h.status else 0;+ lp.metrics.http_status.incr(http.statusCategory(status));+ lp.metrics.http_response_size_bytes.observe(body.len);+ }+try self._events.ensureUnusedCapacity(self.arena, 4);self._events.appendAssumeCapacity(.start);self._events.appendAssumeCapacity(.header);@@ -2520,6 +2560,7 @@ pub const Transfer = struct {log.err(.http, "getResponseCode", .{ .err = err, .source = "body callback" });return http.writefunc_error;};+// Only skip the body when the response will actually be retried// as a redirect (a redirect status with a Location header). Any// other 3xx is a final response whose body must be kept.diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zigindex 6767a00ec..bafe81fe3 100644--- a/src/network/RobotsGate.zig+++ b/src/network/RobotsGate.zig@@ -27,6 +27,7 @@ const lp = @import("lightpanda");const URL = @import("../browser/URL.zig");const ArenaPool = @import("../ArenaPool.zig");+const http = @import("http.zig");const Robots = @import("Robots.zig");const Network = @import("Network.zig");const Transfer = @import("HttpClient.zig").Transfer;@@ -172,6 +173,7 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {} else true;if (!allowed) {+ lp.metrics.robots_access.incr(.deny);log.warn(.http, "blocked by robots", .{ .url = transfer.req.url });transfer.failAsync(error.RobotsBlocked);continue;@@ -179,6 +181,7 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {// Hand back to the pipeline; the robots gate is the last step// before the network. If it fails while we still own the transfer,// clean up here.+ lp.metrics.robots_access.incr(.allow);transfer.client.resumeAfterRobots(transfer) catch |e| {transfer.abortPipelineError(e);};@@ -208,6 +211,7 @@ const RobotsContext = struct {log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });self.status = hdr.status;}+ lp.metrics.robots_status.incr(http.statusCategory(self.status));if (transfer.getContentLength()) |cl| {try self.buffer.ensureTotalCapacity(self.arena, cl);}diff --git a/src/network/http.zig b/src/network/http.zigindex b05206efa..ec3567f70 100644--- a/src/network/http.zig+++ b/src/network/http.zig@@ -624,6 +624,13 @@ pub const Connection = struct {return @intCast(count);}+ // Total transfer time (name lookup to completion) in microseconds.+ pub fn getTotalTimeMicros(self: *const Connection) !c_long {+ var micros: c_long = undefined;+ try libcurl.curl_easy_getinfo(self._easy, .total_time_t, µs);+ return micros;+ }+pub fn getConnectHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue {var hdr: ?*libcurl.CurlHeader = null;libcurl.curl_easy_header(self._easy, name, index, .connect, -1, &hdr) catch |err| {@@ -757,6 +764,62 @@ pub const Handles = struct {}};+pub const StatusCategory = enum {+ @"2xx",+ @"4xx",+ @"5xx",+ other,+};++pub fn statusCategory(status: u16) StatusCategory {+ return switch (status) {+ 200...299 => .@"2xx",+ 400...499 => .@"4xx",+ 500...599 => .@"5xx",+ else => .other,+ };+}++// Coarse failure classes for the http_error metric. Anything we don't+// explicitly bucket lands in `other`.+pub const ErrorReason = enum {+ timeout,+ connect,+ tls,+ too_large,+ aborted,+ robots_blocked,+ other,+};++pub fn errorReason(err: anyerror) ErrorReason {+ return switch (err) {+ error.OperationTimedout => .timeout,+ error.CouldntConnect,+ error.CouldntResolveHost,+ error.CouldntResolveProxy,+ error.NoConnectionAvailable,+ => .connect,+ error.SslConnectError,+ error.PeerFailedVerification,+ error.SslCertproblem,+ error.SslCacertBadfile,+ error.SslIssuerError,+ error.SslPinnedpubkeynotmatch,+ error.SslInvalidcertstatus,+ error.UseSslFailed,+ => .tls,+ error.ResponseTooLarge => .too_large,+ error.Abort,+ error.AbortedByCallback,+ error.AbortAuthChallenge,+ error.SyncWaitInterrupted,+ => .aborted,+ error.RobotsBlocked => .robots_blocked,+ else => .other,+ };+}+fn debugCallback(_: *libcurl.Curl, msg_type: libcurl.CurlInfoType, raw: [*c]u8, len: usize, _: ?*anyopaque) callconv(.c) c_int {const data = raw[0..len];switch (msg_type) {diff --git a/src/sys/libcurl.zig b/src/sys/libcurl.zigindex e473d5896..ae74f82e7 100644--- a/src/sys/libcurl.zig+++ b/src/sys/libcurl.zig@@ -245,6 +245,7 @@ pub const CurlInfo = enum(c.CURLINFO) {redirect_count = c.CURLINFO_REDIRECT_COUNT,response_code = c.CURLINFO_RESPONSE_CODE,connect_code = c.CURLINFO_HTTP_CONNECTCODE,+ total_time_t = c.CURLINFO_TOTAL_TIME_T,};pub const Error = error{@@ -664,6 +665,10 @@ pub fn curl_easy_getinfo(easy: *Curl, comptime info: CurlInfo, out: anytype) Errconst p: *c_long = out;break :blk c.curl_easy_getinfo(easy, inf, p);},+ .total_time_t => blk: {+ const p: *c.curl_off_t = out;+ break :blk c.curl_easy_getinfo(easy, inf, p);+ },.private => blk: {const p: **anyopaque = out;break :blk c.curl_easy_getinfo(easy, inf, p);