Skip to content

Lightpanda: the headless browser designed for AI and automation

`HttpClient`: remove `pending_low_queue`, separate keys for `headers_only` in cache, more eager connection reuse

Halil Durak87a87ba62e70
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 504c6e3e5..f1a50d102 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -539,7 +539,7 @@ pub fn activity(self: *const Client) Activity {
.http = self.http_active + self.dispatch_count + self.intercepted + self.delayed_count,
.ws_events = self.ws_dispatch_count,
.ws_conns = self.ws_active,
- .pending = self.pending_queue.first != null or self.pending_low_queue.first != null,
+ .pending = self.pending_queue.first != null,
};
}
@@ -831,24 +831,13 @@ fn isGated(self: *const Client, transfer: *const Transfer) bool {
return transfer.id != blocking_id;
}
-// Resources the page's progress doesn't depend on. Images are fetched for
-// their status (so load/error is honest) and, with `--fetch-images headers`,
-// a page can queue dozens of them in one parse — none of which should come
-// ahead of the script that's blocking the parser.
-fn isLowPriority(resource_type: Request.ResourceType) bool {
- return switch (resource_type) {
- .image => true,
- .document, .xhr, .script, .fetch, .stylesheet, .eventsource => false,
- };
-}
-
fn startPending(self: *Client) !void {
try self.startDelayed();
while (self.pending_queue.popFirst()) |queue_node| {
const transfer: *Transfer = @fieldParentPtr("_node", queue_node);
const conn = self.network.getConnection() orelse {
- queue.prepend(queue_node);
- return false;
+ self.pending_queue.prepend(queue_node);
+ return;
};
// Bridge state to .created so a failure inside makeRequest before
// any commit cleans up via the failAsync below. makeRequest flips to
@@ -864,7 +853,6 @@ fn startPending(self: *Client) !void {
return err;
};
}
- return true;
}
// Enter the pipeline for every delayed transfer whose time has come.
@@ -1018,19 +1006,24 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
return false;
}
+ const arena = transfer.arena;
+
// Redirects rewrite req.url; the entry must be stored/renewed under the
// URL this lookup ran against, not the final hop. req.url is arena-owned,
// so the captured slice outlives any redirect rewrite.
- transfer._cache_key = req.url;
+ const key: [:0]const u8 = if (req.headers_only)
+ try std.fmt.allocPrintSentinel(arena.allocator(), "headers-only:{s}", .{req.url}, 0)
+ else
+ req.url;
+ transfer._cache_key = key;
- const arena = transfer.arena;
const req_headers = try arena.alloc(http.Header, transfer.req_headers.items.len);
for (transfer.req_headers.items, req_headers) |hdr, *out| {
out.* = .{ .name = hdr.name, .value = hdr.value };
}
const cache_result = cache.get(arena.allocator(), .{
- .url = req.url,
+ .url = key,
.timestamp = lp.datetime.timestamp(.real),
.request_headers = req_headers,
}) catch |e| blk: {
@@ -1046,7 +1039,7 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
},
.revalidate => |cached| {
log.debug(.cache, "revalidate cache entry", .{
- .url = req.url,
+ .url = key,
.etag = cached.etag,
.last_modified = cached.last_modified,
});
@@ -1066,7 +1059,7 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
},
.stale => {
lp.metrics.http_cache.incr(.miss);
- cache.evict(req.url);
+ cache.evict(key);
transfer._cache_intent = .store;
return false;
},
@@ -1119,13 +1112,6 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {
// entry a second time on any early return below.
transfer._cache_intent = .none;
- // A headers_only transfer never read the body. Storing it would put an
- // empty entry under the real cache key and every later full fetch of
- // that URL (an XHR, a `full` image fetch) would hit it and get nothing.
- if (transfer.req.headers_only) {
- return;
- }
-
// could have been disabled while waiting of the response
const cache = self.cache.active() orelse return;
@@ -1304,19 +1290,11 @@ pub fn syncRequest(self: *Client, transfer: *Transfer) !SyncResponse {
}
fn processTransfer(self: *Client, transfer: *Transfer) !void {
- const low = isLowPriority(transfer.req.resource_type);
- if (!low or self.pending_queue.first == null) {
- if (self.network.getConnection()) |conn| {
- return self.makeRequest(conn, transfer);
- }
+ if (self.network.getConnection()) |conn| {
+ return self.makeRequest(conn, transfer);
}
- transfer._queued_low = low;
- if (low) {
- self.pending_low_queue.append(&transfer._node);
- } else {
- self.pending_queue.append(&transfer._node);
- }
+ self.pending_queue.append(&transfer._node);
transfer.state = .queued;
}
@@ -1532,15 +1510,15 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
// we match that behavior: when CURLE_WRITE_ERROR arrives but our callback
// never errored and bytes were received, treat it as success.
const effective_err: ?anyerror = if (msg.err) |err| blk: {
- if (err == error.WriteError and transfer.res.callback_error == null and transfer.res.bytes_received > 0) {
- log.debug(.http, "WriteError downgraded", .{ .url = transfer.req.url, .bytes = transfer.res.bytes_received });
- break :blk null;
- }
// Our own headers_only abort, not a failure: fall through so the
// response is materialized and delivered with an empty body.
if (err == error.WriteError and transfer.res.headers_only_abort) {
break :blk null;
}
+ if (err == error.WriteError and transfer.res.callback_error == null and transfer.res.bytes_received > 0) {
+ log.debug(.http, "WriteError downgraded", .{ .url = transfer.req.url, .bytes = transfer.res.bytes_received });
+ break :blk null;
+ }
break :blk err;
} else null;
@@ -1806,11 +1784,11 @@ pub const Request = struct {
// internal requests transparently following redirects.
pub const RedirectMode = enum { follow, manual, @"error" };
- // Largest body a headers_only transfer will read to the end rather than
- // abort. Draining costs bandwidth but keeps the connection poolable;
- // aborting saves bandwidth but forces a reconnect. 16 KiB is the rough
- // break-even: about ten segments, versus a TCP handshake plus a TLS one.
- pub const HEADERS_ONLY_DRAIN_MAX: usize = 16 * 1024;
+ // How much of a headers_only body we'll read rather than abort. Draining
+ // costs bandwidth but keeps the connection poolable; aborting saves
+ // bandwidth but forces a reconnect. 16 KiB is the rough break-even: about
+ // ten segments, versus a TCP handshake plus a TLS one.
+ const HEADERS_ONLY_DRAIN_MAX: usize = 16 * 1024;
frame_id: u32,
loader_id: u32,
@@ -1827,14 +1805,13 @@ pub const Request = struct {
timeout_ms: u32 = 0,
skip_cache: bool = false,
- // Tear the transfer off the wire as soon as the first body byte arrives:
- // the caller wants the status and the response headers, not the body.
+ // The caller wants the status and the response headers, not the body.
// Unlike a HEAD, the request itself is byte-for-byte a normal GET, so
- // origins and CDNs see (and answer) exactly what a real browser sends.
- // The consumer still gets the usual start/header/done sequence with an
- // empty body; `data_callback` never fires. Note the cost: aborting
- // mid-response means the connection can't be drained, so libcurl closes
- // it instead of returning it to the pool.
+ // origins and CDNs see (and answer) exactly what a real browser sends;
+ // the body is then discarded, and torn off the wire if it doesn't fit in
+ // HEADERS_ONLY_DRAIN_MAX. The consumer still gets the usual
+ // start/header/done sequence, with an empty body; `data_callback` never
+ // fires.
headers_only: bool = false,
// The document frame this request belongs to, for CDP attribution.
@@ -2130,11 +2107,6 @@ pub const Transfer = struct {
// node is always free by then).
_node: std.DoublyLinkedList.Node = .{},
- // Which of the two pending queues _node is linked into. Only meaningful
- // while state == .queued; set at enqueue, read when unlinking, because
- // removing from the wrong list would corrupt both.
- _queued_low: bool = false,
-
// Buffered response ordered events awaiting dispatch.
_events: std.ArrayList(Event) = .empty,
@@ -3115,36 +3087,14 @@ pub const Transfer = struct {
return @intCast(chunk_len);
}
- if (transfer.req.headers_only) {
- const drainable = if (transfer.getContentLength()) |cl|
- cl <= Request.HEADERS_ONLY_DRAIN_MAX
- else
- // No Content-Length (chunked): we can't tell how much is
- // coming, so don't gamble on it being small.
- false;
-
- if (drainable) {
- // Reuses the redirect machinery: consumed, never buffered,
- // so the response still completes with an empty body.
- res.skip_body = true;
- return @intCast(chunk_len);
- }
-
- // Returning writefunc_error is the only way to end a transfer
- // early from a write callback; processOneMessage recognises
- // the flag and treats the resulting CURLE_WRITE_ERROR as a
- // completed response with an empty body.
- res.headers_only_abort = true;
- return http.writefunc_error;
- }
-
- // Pre-size buffer from Content-Length.
- if (transfer.getContentLength()) |cl| {
- if (cl > transfer.client.max_response_size) {
- res.callback_error = error.ResponseTooLarge;
- return http.writefunc_error;
+ if (transfer.req.headers_only == false) {
+ if (transfer.getContentLength()) |cl| {
+ if (cl > transfer.client.max_response_size) {
+ res.callback_error = error.ResponseTooLarge;
+ return http.writefunc_error;
+ }
+ res.buffer.ensureTotalCapacityPrecise(transfer.arena.allocator(), cl) catch {};
}
- res.buffer.ensureTotalCapacityPrecise(transfer.arena.allocator(), cl) catch {};
}
}
@@ -3154,6 +3104,21 @@ pub const Transfer = struct {
res.bytes_received += chunk_len;
+ if (transfer.req.headers_only) {
+ // Plenty of images have no Content-Length to decide this up front, so
+ // decide it as the body arrives.
+ if (res.bytes_received <= Request.HEADERS_ONLY_DRAIN_MAX) {
+ return @intCast(chunk_len);
+ }
+
+ // Returning writefunc_error is the only way to end a transfer
+ // early from a write callback; processOneMessage recognises the
+ // flag and treats the resulting CURLE_WRITE_ERROR as a completed
+ // response with an empty body.
+ res.headers_only_abort = true;
+ return http.writefunc_error;
+ }
+
const chunk = buffer[0..chunk_len];
if (transfer.req.streaming) {