Lightpanda: the headless browser designed for AI and automation
`HttpClient`: remove `pending_low_queue`, separate keys for `headers_only` in cache, more eager connection reuse
Halil Durak87a87ba62e70
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zigindex 504c6e3e5..f1a50d102 100644--- a/src/network/HttpClient.zig+++ b/src/network/HttpClient.zig@@ -539,7 +539,7 @@ pub fn activity(self: *const Client) Activity {.http = self.http_active + self.dispatch_count + self.intercepted + self.delayed_count,.ws_events = self.ws_dispatch_count,.ws_conns = self.ws_active,- .pending = self.pending_queue.first != null or self.pending_low_queue.first != null,+ .pending = self.pending_queue.first != null,};}@@ -831,24 +831,13 @@ fn isGated(self: *const Client, transfer: *const Transfer) bool {return transfer.id != blocking_id;}-// Resources the page's progress doesn't depend on. Images are fetched for-// their status (so load/error is honest) and, with `--fetch-images headers`,-// a page can queue dozens of them in one parse — none of which should come-// ahead of the script that's blocking the parser.-fn isLowPriority(resource_type: Request.ResourceType) bool {- return switch (resource_type) {- .image => true,- .document, .xhr, .script, .fetch, .stylesheet, .eventsource => false,- };-}-fn startPending(self: *Client) !void {try self.startDelayed();while (self.pending_queue.popFirst()) |queue_node| {const transfer: *Transfer = @fieldParentPtr("_node", queue_node);const conn = self.network.getConnection() orelse {- queue.prepend(queue_node);- return false;+ self.pending_queue.prepend(queue_node);+ return;};// Bridge state to .created so a failure inside makeRequest before// any commit cleans up via the failAsync below. makeRequest flips to@@ -864,7 +853,6 @@ fn startPending(self: *Client) !void {return err;};}- return true;}// Enter the pipeline for every delayed transfer whose time has come.@@ -1018,19 +1006,24 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {return false;}+ const arena = transfer.arena;+// Redirects rewrite req.url; the entry must be stored/renewed under the// URL this lookup ran against, not the final hop. req.url is arena-owned,// so the captured slice outlives any redirect rewrite.- transfer._cache_key = req.url;+ const key: [:0]const u8 = if (req.headers_only)+ try std.fmt.allocPrintSentinel(arena.allocator(), "headers-only:{s}", .{req.url}, 0)+ else+ req.url;+ transfer._cache_key = key;- const arena = transfer.arena;const req_headers = try arena.alloc(http.Header, transfer.req_headers.items.len);for (transfer.req_headers.items, req_headers) |hdr, *out| {out.* = .{ .name = hdr.name, .value = hdr.value };}const cache_result = cache.get(arena.allocator(), .{- .url = req.url,+ .url = key,.timestamp = lp.datetime.timestamp(.real),.request_headers = req_headers,}) catch |e| blk: {@@ -1046,7 +1039,7 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {},.revalidate => |cached| {log.debug(.cache, "revalidate cache entry", .{- .url = req.url,+ .url = key,.etag = cached.etag,.last_modified = cached.last_modified,});@@ -1066,7 +1059,7 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {},.stale => {lp.metrics.http_cache.incr(.miss);- cache.evict(req.url);+ cache.evict(key);transfer._cache_intent = .store;return false;},@@ -1119,13 +1112,6 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {// entry a second time on any early return below.transfer._cache_intent = .none;- // A headers_only transfer never read the body. Storing it would put an- // empty entry under the real cache key and every later full fetch of- // that URL (an XHR, a `full` image fetch) would hit it and get nothing.- if (transfer.req.headers_only) {- return;- }-// could have been disabled while waiting of the responseconst cache = self.cache.active() orelse return;@@ -1304,19 +1290,11 @@ pub fn syncRequest(self: *Client, transfer: *Transfer) !SyncResponse {}fn processTransfer(self: *Client, transfer: *Transfer) !void {- const low = isLowPriority(transfer.req.resource_type);- if (!low or self.pending_queue.first == null) {- if (self.network.getConnection()) |conn| {- return self.makeRequest(conn, transfer);- }+ if (self.network.getConnection()) |conn| {+ return self.makeRequest(conn, transfer);}- transfer._queued_low = low;- if (low) {- self.pending_low_queue.append(&transfer._node);- } else {- self.pending_queue.append(&transfer._node);- }+ self.pending_queue.append(&transfer._node);transfer.state = .queued;}@@ -1532,15 +1510,15 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T// we match that behavior: when CURLE_WRITE_ERROR arrives but our callback// never errored and bytes were received, treat it as success.const effective_err: ?anyerror = if (msg.err) |err| blk: {- if (err == error.WriteError and transfer.res.callback_error == null and transfer.res.bytes_received > 0) {- log.debug(.http, "WriteError downgraded", .{ .url = transfer.req.url, .bytes = transfer.res.bytes_received });- break :blk null;- }// Our own headers_only abort, not a failure: fall through so the// response is materialized and delivered with an empty body.if (err == error.WriteError and transfer.res.headers_only_abort) {break :blk null;}+ if (err == error.WriteError and transfer.res.callback_error == null and transfer.res.bytes_received > 0) {+ log.debug(.http, "WriteError downgraded", .{ .url = transfer.req.url, .bytes = transfer.res.bytes_received });+ break :blk null;+ }break :blk err;} else null;@@ -1806,11 +1784,11 @@ pub const Request = struct {// internal requests transparently following redirects.pub const RedirectMode = enum { follow, manual, @"error" };- // Largest body a headers_only transfer will read to the end rather than- // abort. Draining costs bandwidth but keeps the connection poolable;- // aborting saves bandwidth but forces a reconnect. 16 KiB is the rough- // break-even: about ten segments, versus a TCP handshake plus a TLS one.- pub const HEADERS_ONLY_DRAIN_MAX: usize = 16 * 1024;+ // How much of a headers_only body we'll read rather than abort. Draining+ // costs bandwidth but keeps the connection poolable; aborting saves+ // bandwidth but forces a reconnect. 16 KiB is the rough break-even: about+ // ten segments, versus a TCP handshake plus a TLS one.+ const HEADERS_ONLY_DRAIN_MAX: usize = 16 * 1024;frame_id: u32,loader_id: u32,@@ -1827,14 +1805,13 @@ pub const Request = struct {timeout_ms: u32 = 0,skip_cache: bool = false,- // Tear the transfer off the wire as soon as the first body byte arrives:- // the caller wants the status and the response headers, not the body.+ // The caller wants the status and the response headers, not the body.// Unlike a HEAD, the request itself is byte-for-byte a normal GET, so- // origins and CDNs see (and answer) exactly what a real browser sends.- // The consumer still gets the usual start/header/done sequence with an- // empty body; `data_callback` never fires. Note the cost: aborting- // mid-response means the connection can't be drained, so libcurl closes- // it instead of returning it to the pool.+ // origins and CDNs see (and answer) exactly what a real browser sends;+ // the body is then discarded, and torn off the wire if it doesn't fit in+ // HEADERS_ONLY_DRAIN_MAX. The consumer still gets the usual+ // start/header/done sequence, with an empty body; `data_callback` never+ // fires.headers_only: bool = false,// The document frame this request belongs to, for CDP attribution.@@ -2130,11 +2107,6 @@ pub const Transfer = struct {// node is always free by then)._node: std.DoublyLinkedList.Node = .{},- // Which of the two pending queues _node is linked into. Only meaningful- // while state == .queued; set at enqueue, read when unlinking, because- // removing from the wrong list would corrupt both.- _queued_low: bool = false,-// Buffered response ordered events awaiting dispatch._events: std.ArrayList(Event) = .empty,@@ -3115,36 +3087,14 @@ pub const Transfer = struct {return @intCast(chunk_len);}- if (transfer.req.headers_only) {- const drainable = if (transfer.getContentLength()) |cl|- cl <= Request.HEADERS_ONLY_DRAIN_MAX- else- // No Content-Length (chunked): we can't tell how much is- // coming, so don't gamble on it being small.- false;-- if (drainable) {- // Reuses the redirect machinery: consumed, never buffered,- // so the response still completes with an empty body.- res.skip_body = true;- return @intCast(chunk_len);- }-- // Returning writefunc_error is the only way to end a transfer- // early from a write callback; processOneMessage recognises- // the flag and treats the resulting CURLE_WRITE_ERROR as a- // completed response with an empty body.- res.headers_only_abort = true;- return http.writefunc_error;- }-- // Pre-size buffer from Content-Length.- if (transfer.getContentLength()) |cl| {- if (cl > transfer.client.max_response_size) {- res.callback_error = error.ResponseTooLarge;- return http.writefunc_error;+ if (transfer.req.headers_only == false) {+ if (transfer.getContentLength()) |cl| {+ if (cl > transfer.client.max_response_size) {+ res.callback_error = error.ResponseTooLarge;+ return http.writefunc_error;+ }+ res.buffer.ensureTotalCapacityPrecise(transfer.arena.allocator(), cl) catch {};}- res.buffer.ensureTotalCapacityPrecise(transfer.arena.allocator(), cl) catch {};}}@@ -3154,6 +3104,21 @@ pub const Transfer = struct {res.bytes_received += chunk_len;+ if (transfer.req.headers_only) {+ // Plenty of images have no Content-Length to decide this up front, so+ // decide it as the body arrives.+ if (res.bytes_received <= Request.HEADERS_ONLY_DRAIN_MAX) {+ return @intCast(chunk_len);+ }++ // Returning writefunc_error is the only way to end a transfer+ // early from a write callback; processOneMessage recognises the+ // flag and treats the resulting CURLE_WRITE_ERROR as a completed+ // response with an empty body.+ res.headers_only_abort = true;+ return http.writefunc_error;+ }+const chunk = buffer[0..chunk_len];if (transfer.req.streaming) {