Skip to content

Lightpanda: the headless browser designed for AI and automation

add http metrics

Karl Seguinf0784de01560
diff --git a/src/Metrics.zig b/src/Metrics.zig
index 096438fa9..e65148ee7 100644
--- a/src/Metrics.zig
+++ b/src/Metrics.zig
@@ -41,6 +41,36 @@ js_heap_size_bytes: Histogram(&.{
256 * 1024 * 1024,
512 * 1024 * 1024,
}) = .{},
+http_requests: CounterEnum("mode", enum { sync, async }) = .{},
+http_status: CounterEnum("category", @import("network/http.zig").StatusCategory) = .{},
+http_error: CounterEnum("reason", @import("network/http.zig").ErrorReason) = .{},
+http_cache: CounterEnum("result", enum { hit, miss, revalidated }) = .{},
+http_redirects: Counter = .{},
+http_duration_ms: Histogram(&.{
+ 5,
+ 10,
+ 25,
+ 50,
+ 100,
+ 250,
+ 500,
+ 1000,
+ 2500,
+ 5000,
+ 10000,
+}) = .{},
+http_response_size_bytes: Histogram(&.{
+ 32 * 1024,
+ 64 * 1024,
+ 128 * 1024,
+ 256 * 1024,
+ 512 * 1024,
+ 1024 * 1024,
+ 2 * 1024 * 1024,
+ 4 * 1024 * 1024,
+}) = .{},
+robots_status: CounterEnum("category", @import("network/http.zig").StatusCategory) = .{},
+robots_access: CounterEnum("result", enum { allow, deny }) = .{},
// Emitted as each metric's "# HELP" line. A field without an entry is a
// compile error.
@@ -56,6 +86,15 @@ const help = .{
.arena_miss = "Arena pool acquisitions that had to allocate a new arena",
.navigate = "Navigations by initiating frame type",
.js_heap_size_bytes = "V8 heap physical size, sampled when a page is closed",
+ .http_requests = "HTTP requests submitted, by dispatch mode (excludes internal requests like robots.txt)",
+ .http_status = "Final HTTP response status category (redirects counted once, at the final hop)",
+ .http_error = "HTTP requests that failed before delivering a response, by cause",
+ .http_cache = "HTTP cache lookups by outcome",
+ .http_redirects = "HTTP redirect hops followed",
+ .http_duration_ms = "HTTP request wall-clock duration in milliseconds",
+ .http_response_size_bytes = "HTTP response body size in bytes",
+ .robots_status = "robots.txt response status",
+ .robots_access = "robots.txt result",
};
pub fn write(self: *const Metrics, writer: *std.Io.Writer) void {
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index a8e2aa221..2220d6b94 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -780,8 +780,13 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {
}
if (self.obey_robots and !transfer.req.internal) {
switch (try self.robots.check(transfer)) {
- .allowed => {},
- .blocked => return transfer.failAsync(error.RobotsBlocked),
+ .allowed => {
+ lp.metrics.robots_access.incr(.allow);
+ },
+ .blocked => {
+ lp.metrics.robots_access.incr(.deny);
+ return transfer.failAsync(error.RobotsBlocked);
+ },
.pending => return,
}
}
@@ -832,17 +837,20 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
.timestamp = std.time.timestamp(),
.request_headers = req_headers.items,
}) orelse {
+ lp.metrics.http_cache.incr(.miss);
transfer._cache_intent = .store;
return false;
};
if (cached.expired == false) {
+ lp.metrics.http_cache.incr(.hit);
try transfer.bufferCached(cached);
return true;
}
if (cached.metadata.hasValidators() == false) {
// Expired and no validators
+ lp.metrics.http_cache.incr(.miss);
cached.data.deinit();
cache.evict(req.url);
transfer._cache_intent = .store;
@@ -889,6 +897,7 @@ fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool {
log.warn(.cache, "renew failed", .{ .err = err });
};
+ lp.metrics.http_cache.incr(.revalidated);
try transfer.bufferCached(stale);
return true;
}
@@ -1021,6 +1030,7 @@ pub fn syncRequest(self: *Client, allocator: Allocator, req: Request) !SyncRespo
errdefer sync_ctx.body.deinit(allocator);
var r = req;
+ r.sync = true;
r.ctx = &sync_ctx;
r.header_callback = SyncContext.headerCallback;
r.data_callback = SyncContext.dataCallback;
@@ -1350,6 +1360,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
if (msg.conn.getResponseHeader("location", 0)) |location| switch (transfer.req.redirect) {
.follow => {
try transfer.handleRedirect(location.value);
+ if (!transfer.req.internal) lp.metrics.http_redirects.incr();
const conn = transfer._conn.?;
@@ -1412,6 +1423,14 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
try transfer.materializeResponse(msg.conn);
+ // Latency is only meaningful for responses that hit the network (cache
+ // and synthetic responses never reach processOneMessage).
+ if (!transfer.req.internal) {
+ if (msg.conn.getTotalTimeMicros()) |micros| {
+ lp.metrics.http_duration_ms.observe(@intCast(@max(0, @divTrunc(micros, 1000))));
+ } else |_| {}
+ }
+
// Release the conn before any of this response's callbacks can run —
// they'll want it for the next resource.
self.removeConn(msg.conn);
@@ -1549,6 +1568,9 @@ pub const Request = struct {
// these do not need to be deferred and do not obey robots.txt.
internal: bool = false,
+ // Set by syncRequest; only used to label the http_requests metric.
+ sync: bool = false,
+
// When false, the caller does not guarantee that the body outlives the
// transfer, and thus we'll need to dupe it.
body_outlives_request: bool = false,
@@ -1926,6 +1948,9 @@ pub const Transfer = struct {
// on failure the transfer is already cleaned up (error_callback fired,
// memory freed) — do not deinit it, and do not touch it after this.
pub fn submit(self: *Transfer) anyerror!void {
+ if (!self.req.internal) {
+ lp.metrics.http_requests.incr(if (self.req.sync) .sync else .async);
+ }
// Synthetic schemes never touch the network — no robots, cache, or
// interception. The response is materialized here and delivered by
// the dispatcher like any other transfer.
@@ -2136,6 +2161,10 @@ pub const Transfer = struct {
}
self._notified_fail = true;
+ if (!self.req.internal) {
+ lp.metrics.http_error.incr(http.errorReason(err));
+ }
+
if (self._notify_cdp) {
self.req.notification.dispatch(.http_request_fail, &.{
.transfer = self,
@@ -2184,6 +2213,17 @@ pub const Transfer = struct {
// Buffer the standard success event sequence. `body` is either owned by
// transfer.arena OR, through some other mechanism, outlives the transfer.
fn bufferEvents(self: *Transfer, body: []const u8) !void {
+ // Single delivery point for every successful response — network,
+ // cache, and synthetic, on both the sync and async paths. Redirect
+ // hops never reach here (they loop back in processOneMessage), so
+ // http_status reflects the final response only. Internal requests
+ // (robots.txt) are tracked by the robots_* metrics instead.
+ if (!self.req.internal) {
+ const status = if (self.res.header) |h| h.status else 0;
+ lp.metrics.http_status.incr(http.statusCategory(status));
+ lp.metrics.http_response_size_bytes.observe(body.len);
+ }
+
try self._events.ensureUnusedCapacity(self.arena, 4);
self._events.appendAssumeCapacity(.start);
self._events.appendAssumeCapacity(.header);
@@ -2520,6 +2560,7 @@ pub const Transfer = struct {
log.err(.http, "getResponseCode", .{ .err = err, .source = "body callback" });
return http.writefunc_error;
};
+
// Only skip the body when the response will actually be retried
// as a redirect (a redirect status with a Location header). Any
// other 3xx is a final response whose body must be kept.
diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig
index 6767a00ec..bafe81fe3 100644
--- a/src/network/RobotsGate.zig
+++ b/src/network/RobotsGate.zig
@@ -27,6 +27,7 @@ const lp = @import("lightpanda");
const URL = @import("../browser/URL.zig");
const ArenaPool = @import("../ArenaPool.zig");
+const http = @import("http.zig");
const Robots = @import("Robots.zig");
const Network = @import("Network.zig");
const Transfer = @import("HttpClient.zig").Transfer;
@@ -172,6 +173,7 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {
} else true;
if (!allowed) {
+ lp.metrics.robots_access.incr(.deny);
log.warn(.http, "blocked by robots", .{ .url = transfer.req.url });
transfer.failAsync(error.RobotsBlocked);
continue;
@@ -179,6 +181,7 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {
// Hand back to the pipeline; the robots gate is the last step
// before the network. If it fails while we still own the transfer,
// clean up here.
+ lp.metrics.robots_access.incr(.allow);
transfer.client.resumeAfterRobots(transfer) catch |e| {
transfer.abortPipelineError(e);
};
@@ -208,6 +211,7 @@ const RobotsContext = struct {
log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });
self.status = hdr.status;
}
+ lp.metrics.robots_status.incr(http.statusCategory(self.status));
if (transfer.getContentLength()) |cl| {
try self.buffer.ensureTotalCapacity(self.arena, cl);
}
diff --git a/src/network/http.zig b/src/network/http.zig
index b05206efa..ec3567f70 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -624,6 +624,13 @@ pub const Connection = struct {
return @intCast(count);
}
+ // Total transfer time (name lookup to completion) in microseconds.
+ pub fn getTotalTimeMicros(self: *const Connection) !c_long {
+ var micros: c_long = undefined;
+ try libcurl.curl_easy_getinfo(self._easy, .total_time_t, &micros);
+ return micros;
+ }
+
pub fn getConnectHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue {
var hdr: ?*libcurl.CurlHeader = null;
libcurl.curl_easy_header(self._easy, name, index, .connect, -1, &hdr) catch |err| {
@@ -757,6 +764,62 @@ pub const Handles = struct {
}
};
+pub const StatusCategory = enum {
+ @"2xx",
+ @"4xx",
+ @"5xx",
+ other,
+};
+
+pub fn statusCategory(status: u16) StatusCategory {
+ return switch (status) {
+ 200...299 => .@"2xx",
+ 400...499 => .@"4xx",
+ 500...599 => .@"5xx",
+ else => .other,
+ };
+}
+
+// Coarse failure classes for the http_error metric. Anything we don't
+// explicitly bucket lands in `other`.
+pub const ErrorReason = enum {
+ timeout,
+ connect,
+ tls,
+ too_large,
+ aborted,
+ robots_blocked,
+ other,
+};
+
+pub fn errorReason(err: anyerror) ErrorReason {
+ return switch (err) {
+ error.OperationTimedout => .timeout,
+ error.CouldntConnect,
+ error.CouldntResolveHost,
+ error.CouldntResolveProxy,
+ error.NoConnectionAvailable,
+ => .connect,
+ error.SslConnectError,
+ error.PeerFailedVerification,
+ error.SslCertproblem,
+ error.SslCacertBadfile,
+ error.SslIssuerError,
+ error.SslPinnedpubkeynotmatch,
+ error.SslInvalidcertstatus,
+ error.UseSslFailed,
+ => .tls,
+ error.ResponseTooLarge => .too_large,
+ error.Abort,
+ error.AbortedByCallback,
+ error.AbortAuthChallenge,
+ error.SyncWaitInterrupted,
+ => .aborted,
+ error.RobotsBlocked => .robots_blocked,
+ else => .other,
+ };
+}
+
fn debugCallback(_: *libcurl.Curl, msg_type: libcurl.CurlInfoType, raw: [*c]u8, len: usize, _: ?*anyopaque) callconv(.c) c_int {
const data = raw[0..len];
switch (msg_type) {
diff --git a/src/sys/libcurl.zig b/src/sys/libcurl.zig
index e473d5896..ae74f82e7 100644
--- a/src/sys/libcurl.zig
+++ b/src/sys/libcurl.zig
@@ -245,6 +245,7 @@ pub const CurlInfo = enum(c.CURLINFO) {
redirect_count = c.CURLINFO_REDIRECT_COUNT,
response_code = c.CURLINFO_RESPONSE_CODE,
connect_code = c.CURLINFO_HTTP_CONNECTCODE,
+ total_time_t = c.CURLINFO_TOTAL_TIME_T,
};
pub const Error = error{
@@ -664,6 +665,10 @@ pub fn curl_easy_getinfo(easy: *Curl, comptime info: CurlInfo, out: anytype) Err
const p: *c_long = out;
break :blk c.curl_easy_getinfo(easy, inf, p);
},
+ .total_time_t => blk: {
+ const p: *c.curl_off_t = out;
+ break :blk c.curl_easy_getinfo(easy, inf, p);
+ },
.private => blk: {
const p: **anyopaque = out;
break :blk c.curl_easy_getinfo(easy, inf, p);