Skip to content

Lightpanda: the headless browser designed for AI and automation

webapi: Add EventSource

Karl Seguin8ad4bd901599
diff --git a/src/browser/Mime.zig b/src/browser/Mime.zig
index 402056079..afc71a1f4 100644
--- a/src/browser/Mime.zig
+++ b/src/browser/Mime.zig
@@ -41,6 +41,7 @@ pub const ContentTypeEnum = enum {
text_plain,
text_css,
text_markdown,
+ text_event_stream,
image_jpeg,
image_gif,
image_png,
@@ -57,6 +58,7 @@ pub const ContentType = union(ContentTypeEnum) {
text_plain: void,
text_css: void,
text_markdown: void,
+ text_event_stream: void,
image_jpeg: void,
image_gif: void,
image_png: void,
@@ -76,6 +78,7 @@ pub fn contentTypeString(mime: *const Mime) []const u8 {
.text_plain => "text/plain",
.text_markdown => "text/markdown",
.text_css => "text/css",
+ .text_event_stream => "text/event-stream",
.image_jpeg => "image/jpeg",
.image_png => "image/png",
.image_gif => "image/gif",
@@ -363,6 +366,7 @@ fn parseContentType(value: []const u8) !struct { ContentType, usize } {
@"text/css",
@"text/plain",
@"text/markdown",
+ @"text/event-stream",
@"text/javascript",
@"application/javascript",
@@ -382,6 +386,7 @@ fn parseContentType(value: []const u8) !struct { ContentType, usize } {
.@"text/plain" => .{ .text_plain = {} },
.@"text/css" => .{ .text_css = {} },
.@"text/markdown" => .{ .text_markdown = {} },
+ .@"text/event-stream" => .{ .text_event_stream = {} },
.@"image/jpeg" => .{ .image_jpeg = {} },
.@"image/png" => .{ .image_png = {} },
.@"image/gif" => .{ .image_gif = {} },
diff --git a/src/browser/js/bridge.zig b/src/browser/js/bridge.zig
index 03bb0bbaf..3a18fbcfb 100644
--- a/src/browser/js/bridge.zig
+++ b/src/browser/js/bridge.zig
@@ -1063,6 +1063,7 @@ pub const PageJsApis = flattenTypes(&.{
@import("../webapi/net/XMLHttpRequestEventTarget.zig"),
@import("../webapi/net/XMLHttpRequestUpload.zig"),
@import("../webapi/net/WebSocket.zig"),
+ @import("../webapi/net/EventSource.zig"),
@import("../webapi/event/CloseEvent.zig"),
@import("../webapi/streams/ReadableStream.zig"),
@import("../webapi/streams/ReadableStreamDefaultReader.zig"),
@@ -1166,6 +1167,7 @@ pub const WorkerJsApis = flattenTypes(&.{
@import("../webapi/net/XMLHttpRequestEventTarget.zig"),
@import("../webapi/net/XMLHttpRequestUpload.zig"),
@import("../webapi/net/WebSocket.zig"),
+ @import("../webapi/net/EventSource.zig"),
@import("../webapi/FileReader.zig"),
@import("../webapi/ImageData.zig"),
@import("../webapi/Performance.zig"),
diff --git a/src/browser/tests/net/eventsource-worker.js b/src/browser/tests/net/eventsource-worker.js
new file mode 100644
index 000000000..86ee614a9
--- /dev/null
+++ b/src/browser/tests/net/eventsource-worker.js
@@ -0,0 +1,18 @@
+// Exercises EventSource inside a worker. Receives a command from the page,
+// consumes the stream, and posts the results back.
+self.onmessage = function() {
+ try {
+ const received = [];
+ const es = new EventSource('http://127.0.0.1:9582/sse/simple');
+ es.onopen = () => received.push('open');
+ es.onmessage = (e) => received.push(e.data);
+ es.addEventListener('custom', (e) => {
+ received.push(e.data);
+ es.close();
+ postMessage({ ok: true, received, readyState: es.readyState });
+ });
+ es.onerror = () => postMessage({ ok: false, err: 'eventsource error' });
+ } catch (err) {
+ postMessage({ ok: false, err: String(err) });
+ }
+};
diff --git a/src/browser/tests/net/eventsource.html b/src/browser/tests/net/eventsource.html
new file mode 100644
index 000000000..ce2855a89
--- /dev/null
+++ b/src/browser/tests/net/eventsource.html
@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+<script src="../testing.js"></script>
+
+<script id=constants type=module>
+ testing.expectEqual(0, EventSource.CONNECTING);
+ testing.expectEqual(1, EventSource.OPEN);
+ testing.expectEqual(2, EventSource.CLOSED);
+
+ let threw = false;
+ try {
+ new EventSource('http://this is invalid/');
+ } catch (e) {
+ threw = true;
+ testing.expectEqual('SyntaxError', e.name);
+ }
+ testing.expectTrue(threw, 'invalid URL must throw');
+</script>
+
+<script id=simple type=module>
+{
+ const state = await testing.async();
+ const received = [];
+ const es = new EventSource('http://127.0.0.1:9582/sse/simple');
+ testing.expectEqual(0, es.readyState);
+ testing.expectEqual('http://127.0.0.1:9582/sse/simple', es.url);
+ testing.expectEqual(false, es.withCredentials);
+
+ es.onopen = () => {
+ received.push(['open', es.readyState]);
+ };
+ es.onmessage = (e) => {
+ received.push(['message', e.data, e.lastEventId, e.origin, e.isTrusted]);
+ };
+ es.addEventListener('custom', (e) => {
+ received.push(['custom', e.data, e.lastEventId]);
+ es.close();
+ received.push(['closed', es.readyState]);
+ state.resolve();
+ });
+
+ await state.done(() => {
+ testing.expectEqual([
+ ['open', 1],
+ ['message', 'first', '', 'http://127.0.0.1:9582', true],
+ ['custom', 'a\nb', '42'],
+ ['closed', 2],
+ ], received);
+ });
+}
+</script>
+
+<script id=streaming type=module>
+{
+ // The server only sends "second" after seeing our /sse/flag request, which
+ // we make on receiving "first" — so this only completes if data events are
+ // delivered while the response is still streaming.
+ const state = await testing.async();
+ const received = [];
+ const es = new EventSource('http://127.0.0.1:9582/sse/streaming');
+ es.onmessage = (e) => {
+ received.push(e.data);
+ if (e.data === 'first') {
+ fetch('http://127.0.0.1:9582/sse/flag');
+ } else {
+ es.close();
+ state.resolve();
+ }
+ };
+
+ await state.done(() => {
+ testing.expectEqual(['first', 'second'], received);
+ });
+}
+</script>
+
+<script id=reconnect type=module>
+{
+ const state = await testing.async();
+ const received = [];
+ const es = new EventSource('http://127.0.0.1:9582/sse/reconnect');
+ es.onerror = () => {
+ received.push(['error', es.readyState]);
+ };
+ es.onmessage = (e) => {
+ received.push(e.data);
+ if (e.data === 'two') {
+ es.close();
+ state.resolve();
+ }
+ };
+
+ await state.done(() => {
+ testing.expectEqual(['one', ['error', 0], 'two'], received);
+ });
+}
+</script>
+
+<script id=pending_reconnect_at_teardown type=module>
+{
+ // Deliberately left open: the stream ends and the source sits on its
+ // (60s) reconnect timer with no transfer, so page teardown can only
+ // clean it up through the scheduler-task finalizer. The test runner's
+ // leak detection fails this test if that path regresses.
+ const state = await testing.async();
+ const es = new EventSource('http://127.0.0.1:9582/sse/long_retry');
+ es.onerror = () => state.resolve(es.readyState);
+
+ await state.done((readyState) => {
+ testing.expectEqual(0, readyState);
+ });
+}
+</script>
+
+<script id=fail_status type=module>
+{
+ const state = await testing.async();
+ const es = new EventSource('http://127.0.0.1:9582/sse/404');
+ es.onmessage = () => { throw new Error('unexpected message'); };
+ es.onerror = () => state.resolve(es.readyState);
+
+ await state.done((readyState) => {
+ testing.expectEqual(2, readyState);
+ });
+}
+</script>
+
+<script id=fail_mime type=module>
+{
+ const state = await testing.async();
+ const es = new EventSource('http://127.0.0.1:9582/sse/badmime');
+ es.onmessage = () => { throw new Error('unexpected message'); };
+ es.onerror = () => state.resolve(es.readyState);
+
+ await state.done((readyState) => {
+ testing.expectEqual(2, readyState);
+ });
+}
+</script>
+
+<script id=fail_scheme type=module>
+{
+ const state = await testing.async();
+ const es = new EventSource('ftp://127.0.0.1/');
+ es.onerror = () => state.resolve(es.readyState);
+
+ await state.done((readyState) => {
+ testing.expectEqual(2, readyState);
+ });
+}
+</script>
diff --git a/src/browser/tests/net/eventsource_worker.html b/src/browser/tests/net/eventsource_worker.html
new file mode 100644
index 000000000..2a7f1e3fe
--- /dev/null
+++ b/src/browser/tests/net/eventsource_worker.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<script src="../testing.js"></script>
+
+<script id=eventsource_in_worker type=module>
+{
+ const state = await testing.async();
+ const worker = new Worker('./eventsource-worker.js');
+ worker.onmessage = (e) => state.resolve(e.data);
+ worker.postMessage({});
+
+ await state.done((data) => {
+ testing.expectTrue(data.ok, 'worker eventsource error: ' + data.err);
+ testing.expectEqual(['open', 'first', 'a\nb'], data.received);
+ testing.expectEqual(2, data.readyState);
+ });
+}
+</script>
diff --git a/src/browser/tests/testing.js b/src/browser/tests/testing.js
index 7adb75427..66004f6b9 100644
--- a/src/browser/tests/testing.js
+++ b/src/browser/tests/testing.js
@@ -78,7 +78,7 @@
async function async() {
const script_id = (IS_TEST_RUNNER) ? document.currentScript.id : 'cannot track module id in FF/Chrome';
- if (async_seen.has(script_id)) {
+ if (async_seen.has(script_id) && IS_TEST_RUNNER) {
throw new Error(`testing.async() called more than once for script '${script_id}'. A script may only register one async block (the runner can declare success in the gap between two of them); split the test into separate <script> tags.`);
}
async_seen.add(script_id);
diff --git a/src/browser/webapi/EventTarget.zig b/src/browser/webapi/EventTarget.zig
index eaf39dbdb..2265822a2 100644
--- a/src/browser/webapi/EventTarget.zig
+++ b/src/browser/webapi/EventTarget.zig
@@ -50,6 +50,7 @@ pub const Type = union(enum) {
file_reader: *@import("FileReader.zig"),
font_face_set: *@import("css/FontFaceSet.zig"),
websocket: *@import("net/WebSocket.zig"),
+ event_source: *@import("net/EventSource.zig"),
cookie_store: *@import("storage/CookieStore.zig"),
idb_request: *@import("storage/idb/IDBRequest.zig"),
idb_database: *@import("storage/idb/IDBDatabase.zig"),
@@ -166,6 +167,7 @@ pub fn format(self: *EventTarget, writer: *std.Io.Writer) !void {
.file_reader => writer.writeAll("<FileReader>"),
.font_face_set => writer.writeAll("<FontFaceSet>"),
.websocket => writer.writeAll("<WebSocket>"),
+ .event_source => writer.writeAll("<EventSource>"),
.cookie_store => writer.writeAll("<CookieStore>"),
.idb_request => writer.writeAll("<IDBRequest>"),
.idb_database => writer.writeAll("<IDBDatabase>"),
@@ -194,6 +196,7 @@ pub fn toString(self: *EventTarget) []const u8 {
.file_reader => return "[object FileReader]",
.font_face_set => return "[object FontFaceSet]",
.websocket => return "[object WebSocket]",
+ .event_source => return "[object EventSource]",
.cookie_store => return "[object CookieStore]",
.idb_request => return "[object IDBRequest]",
.idb_database => return "[object IDBDatabase]",
diff --git a/src/browser/webapi/event/MessageEvent.zig b/src/browser/webapi/event/MessageEvent.zig
index 9ed49ce1b..02fb1fe83 100644
--- a/src/browser/webapi/event/MessageEvent.zig
+++ b/src/browser/webapi/event/MessageEvent.zig
@@ -35,12 +35,14 @@ const MessageEvent = @This();
_proto: *Event,
_data: ?Data = null,
_origin: []const u8 = "",
+_last_event_id: []const u8 = "",
_source: ?*Window = null,
_ports: []const *MessagePort = &.{},
const MessageEventOptions = struct {
data: ?Data = null,
origin: ?[]const u8 = null,
+ lastEventId: ?[]const u8 = null,
source: ?*Window = null,
ports: []const *MessagePort = &.{},
};
@@ -77,6 +79,7 @@ fn initWithTrusted(arena: Allocator, typ: String, opts_: ?Options, trusted: bool
._proto = undefined,
._data = opts.data,
._origin = if (opts.origin) |str| try arena.dupe(u8, str) else "",
+ ._last_event_id = if (opts.lastEventId) |str| try arena.dupe(u8, str) else "",
._source = opts.source,
._ports = if (opts.ports.len == 0) &.{} else try arena.dupe(*MessagePort, opts.ports),
},
@@ -117,6 +120,10 @@ pub fn getOrigin(self: *const MessageEvent) []const u8 {
return self._origin;
}
+pub fn getLastEventId(self: *const MessageEvent) []const u8 {
+ return self._last_event_id;
+}
+
pub fn getSource(self: *const MessageEvent, exec: *js.Execution) ?Window.Access {
switch (exec.js.global) {
.frame => |frame| {
@@ -149,6 +156,7 @@ pub const JsApi = struct {
pub const constructor = bridge.constructor(MessageEvent.init, .{});
pub const data = bridge.accessor(MessageEvent.getData, null, .{});
pub const origin = bridge.accessor(MessageEvent.getOrigin, null, .{});
+ pub const lastEventId = bridge.accessor(MessageEvent.getLastEventId, null, .{});
pub const source = bridge.accessor(MessageEvent.getSource, null, .{});
pub const ports = bridge.accessor(MessageEvent.getPorts, null, .{});
};
diff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zig
new file mode 100644
index 000000000..6244695a0
--- /dev/null
+++ b/src/browser/webapi/net/EventSource.zig
@@ -0,0 +1,661 @@
+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <[email protected]>
+// Pierre Tachoire <[email protected]>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const lp = @import("lightpanda");
+
+const js = @import("../../js/js.zig");
+
+const URL = @import("../../URL.zig");
+const Mime = @import("../../Mime.zig");
+const Page = @import("../../Page.zig");
+const Transfer = @import("../../../network/HttpClient.zig").Transfer;
+
+const Event = @import("../Event.zig");
+const EventTarget = @import("../EventTarget.zig");
+const MessageEvent = @import("../event/MessageEvent.zig");
+
+const log = lp.log;
+const String = lp.String;
+const Execution = js.Execution;
+const Allocator = std.mem.Allocator;
+const IS_DEBUG = @import("builtin").mode == .Debug;
+
+// https://html.spec.whatwg.org/multipage/server-sent-events.html
+const EventSource = @This();
+
+_rc: lp.RC(u16) = .{},
+_exec: *const Execution,
+_proto: *EventTarget,
+_arena: Allocator,
+
+_url: [:0]const u8,
+_with_credentials: bool = false,
+_ready_state: ReadyState = .connecting,
+_transfer: ?*Transfer = null,
+_max_response_size: usize,
+
+// Holds a self-reference from creation until close() / a failed connection
+// / owner teardown. Guards the double-release in deactivate().
+_active: bool = true,
+
+// The committed "last event ID string": stamped on every message event and
+// sent as the Last-Event-ID header on reconnects. A reused buffer — ids
+// change on most events, one arena dupe per commit would grow forever.
+_last_event_id: std.ArrayList(u8) = .empty,
+
+_event_origin: []const u8 = "",
+
+_reconnect_ms: u32 = 3000,
+
+// Per-stream parser state, reset by connect().
+_line_buf: std.ArrayList(u8) = .empty,
+_data_buf: std.ArrayList(u8) = .empty,
+_event_type_buf: std.ArrayList(u8) = .empty,
+
+// The "last event ID buffer": committed to _last_event_id when an event
+// block completes (even one that dispatches nothing).
+_id_buf: std.ArrayList(u8) = .empty,
+
+// if we ended on a CR, then we should skip a leading LF
+_skip_lf: bool = false,
+
+// A single leading BOM is stripped from the stream's first line.
+_bom_checked: bool = false,
+
+_on_open: ?js.Function.Temp = null,
+_on_message: ?js.Function.Temp = null,
+_on_error: ?js.Function.Temp = null,
+
+pub const ReadyState = enum(u8) {
+ connecting = 0,
+ open = 1,
+ closed = 2,
+};
+
+const Opts = struct {
+ withCredentials: bool = false,
+};
+
+pub fn init(url: []const u8, opts_: ?Opts, exec: *const Execution) !*EventSource {
+ const arena = try exec.getArena(.medium, "EventSource");
+ errdefer exec.releaseArena(arena);
+
+ const resolved = URL.resolve(arena, exec.base(), url, .{ .encoding = exec.charset.* }) catch {
+ return error.SyntaxError;
+ };
+
+ const self = try exec._factory.eventTargetWithAllocator(arena, EventSource{
+ ._exec = exec,
+ ._arena = arena,
+ ._proto = undefined,
+ ._url = resolved,
+ ._with_credentials = if (opts_) |o| o.withCredentials else false,
+ ._max_response_size = exec.session.browser.http_client.max_response_size,
+ });
+
+ // deactivate() releases it.
+ self.acquireRef();
+
+ if (comptime IS_DEBUG) {
+ log.debug(.http, "EventSource connecting", .{ .url = resolved });
+ }
+
+ if (std.ascii.startsWithIgnoreCase(url, "http://") == false and std.ascii.startsWithIgnoreCase(url, "https://") == false) {
+ try self.scheduleTask(failConnection, 0, "EventSource.fail");
+ return self;
+ }
+
+ self.connect() catch |err| {
+ log.info(.http, "EventSource connect", .{ .err = err });
+ try self.scheduleTask(failConnection, 0, "EventSource.fail");
+ };
+ return self;
+}
+
+pub fn deinit(self: *EventSource, page: *Page) void {
+ self._ready_state = .closed;
+ if (self._transfer) |transfer| {
+ self._transfer = null;
+ transfer.abort(error.Abort);
+ }
+
+ if (self._on_open) |func| {
+ func.release();
+ }
+ if (self._on_message) |func| {
+ func.release();
+ }
+ if (self._on_error) |func| {
+ func.release();
+ }
+
+ page.releaseArena(self._arena);
+}
+
+pub fn releaseRef(self: *EventSource, page: *Page) void {
+ self._rc.release(self, page);
+}
+
+pub fn acquireRef(self: *EventSource) void {
+ self._rc.acquire();
+}
+
+fn asEventTarget(self: *EventSource) *EventTarget {
+ return self._proto;
+}
+
+fn connect(self: *EventSource) !void {
+ const exec = self._exec;
+ const session = exec.session;
+ const http_client = &session.browser.http_client;
+
+ self._skip_lf = false;
+ self._bom_checked = false;
+ self._line_buf.clearRetainingCapacity();
+ self._data_buf.clearRetainingCapacity();
+ self._event_type_buf.clearRetainingCapacity();
+
+ self._id_buf.clearRetainingCapacity();
+ try self._id_buf.appendSlice(self._arena, self._last_event_id.items);
+
+ var headers = try http_client.newHeaders();
+ try headers.add("Accept: text/event-stream");
+ try headers.add("Cache-Control: no-cache");
+ if (self._last_event_id.items.len > 0) {
+ // headers.add copies the value, so the local arena's lifetime is enough
+ const header = try std.fmt.allocPrintSentinel(exec.local_arena, "Last-Event-ID: {s}", .{self._last_event_id.items}, 0);
+ try headers.add(header);
+ }
+
+ const same_origin = exec.isSameOrigin(self._url);
+ if (!same_origin) {
+ // EventSource is a CORS request: cross-origin fetches carry the
+ // document's origin ("null" for opaque origins, like Chrome).
+ const origin = exec.origin() orelse "null";
+ const header = try std.fmt.allocPrintSentinel(exec.local_arena, "Origin: {s}", .{origin}, 0);
+ try headers.add(header);
+ }
+ try exec.headersForRequest(&headers);
+
+ const cookie_support = self._with_credentials or same_origin;
+
+ const transfer = try exec.newRequest(.{
+ .ctx = self,
+ .url = self._url,
+ .method = .GET,
+ .headers = headers,
+ .frame_id = exec.frameId(),
+ .loader_id = exec.loaderId(),
+ .cookie_jar = if (cookie_support) &session.cookie_jar else null,
+ .cookie_origin = exec.url.*,
+ .resource_type = .eventsource,
+ .streaming = true,
+ .notification = session.notification,
+ .header_callback = httpHeaderDoneCallback,
+ .data_callback = httpDataCallback,
+ .done_callback = httpDoneCallback,
+ .error_callback = httpErrorCallback,
+ .shutdown_callback = httpShutdownCallback,
+ });
+
+ self._transfer = transfer;
+
+ // Failures inside submit are dispatched to httpErrorCallback.
+ transfer.submit() catch {};
+}
+
+fn reconnectTask(self: *EventSource) void {
+ if (self._ready_state != .connecting or self._transfer != null) {
+ // closed while the timer was pending
+ return;
+ }
+ self.connect() catch |err| {
+ log.warn(.http, "EventSource reconnect", .{ .err = err, .url = self._url });
+ self.failConnection();
+ };
+}
+
+pub fn close(self: *EventSource) void {
+ if (self._ready_state == .closed) {
+ return;
+ }
+ self._ready_state = .closed;
+ self.deactivate();
+}
+
+fn deactivate(self: *EventSource) void {
+ if (!self._active) {
+ return;
+ }
+ self._active = false;
+ if (self._transfer) |transfer| {
+ self._transfer = null;
+ transfer.abort(error.Abort);
+ }
+ self.releaseRef(self._exec.page);
+}
+
+fn failConnection(self: *EventSource) void {
+ if (self._ready_state == .closed) {
+ return;
+ }
+ self._ready_state = .closed;
+ self.dispatchEvent("error", self._on_error) catch |err| {
+ log.err(.http, "EventSource error event", .{ .err = err, .url = self._url });
+ };
+ self.deactivate();
+}
+
+fn reestablish(self: *EventSource) void {
+ if (self._ready_state == .closed) {
+ return;
+ }
+ self._ready_state = .connecting;
+ self.dispatchEvent("error", self._on_error) catch |err| {
+ log.err(.http, "EventSource error event", .{ .err = err, .url = self._url });
+ };
+ // the error handler may have close()d us
+ if (self._ready_state == .closed) {
+ return;
+ }
+ self.scheduleTask(reconnectTask, self._reconnect_ms, "EventSource.reconnect") catch {
+ self.failConnection();
+ };
+}
+
+// runs the task while ensuring self is kept alive
+fn scheduleTask(self: *EventSource, comptime task: fn (*EventSource) void, ms: u32, name: []const u8) !void {
+ const Wrapper = struct {
+ fn run(ctx: *anyopaque) anyerror!?u32 {
+ const es: *EventSource = @ptrCast(@alignCast(ctx));
+ defer es.releaseRef(es._exec.page);
+ task(es);
+ return null;
+ }
+ fn finalize(ctx: *anyopaque) void {
+ // Scheduler teardown with the task still pending: the context is
+ // going away. A source waiting on this task holds no transfer, so
+ // no shutdown_callback will ever deactivate it — do it here (must
+ // not run JS), then drop the task's own reference.
+ const es: *EventSource = @ptrCast(@alignCast(ctx));
+ es._ready_state = .closed;
+ es.deactivate();
+ es.releaseRef(es._exec.page);
+ }
+ };
+
+ self.acquireRef();
+ errdefer self.releaseRef(self._exec.page);
+ try self._exec._scheduler.add(self, Wrapper.run, ms, .{
+ .name = name,
+ .finalizer = Wrapper.finalize,
+ });
+}
+
+fn httpHeaderDoneCallback(transfer: *Transfer) !Transfer.HeaderResult {
+ const self: *EventSource = @ptrCast(@alignCast(transfer.req.ctx));
+
+ const status = transfer.responseStatus().?;
+ const mime_ok = blk: {
+ const ct = transfer.contentType() orelse break :blk false;
+ const mime = Mime.parse(ct) catch break :blk false;
+ break :blk mime.content_type == .text_event_stream;
+ };
+
+ if (comptime IS_DEBUG) {
+ log.debug(.http, "request header", .{
+ .source = "eventsource",
+ .url = self._url,
+ .status = status,
+ .mime_ok = mime_ok,
+ });
+ }
+
+ if (status != 200 or !mime_ok) {
+ self.failConnection();
+ return .abort;
+ }
+
+ if (!self._exec.isSameOrigin(self._url) and !self.corsAllowed(transfer)) {
+ self.failConnection();
+ return .abort;
+ }
+
+ var ls: js.Local.Scope = undefined;
+ self._exec.js.localScope(&ls);
+ defer ls.deinit();
+
+ const final_url = try self._arena.dupeZ(u8, transfer.req.url);
+ self._event_origin = (URL.getOrigin(self._arena, final_url) catch null) orelse "";
+
+ // https://html.spec.whatwg.org/multipage/server-sent-events.html#announce-the-connection
+ self._ready_state = .open;
+ try self.dispatchEvent("open", self._on_open);
+ return .proceed;
+}
+
+// The general http stack doesn't model CORS (fetch/XHR assume it passed),
+// but the EventSource WPTs require enforcement, and it's self-contained
+// enough to do here: the response must echo an acceptable
+// Access-Control-Allow-Origin (plus Allow-Credentials for credentialed
+// requests).
+fn corsAllowed(self: *const EventSource, transfer: *Transfer) bool {
+ var allow_origin: ?[]const u8 = null;
+ var allow_credentials: ?[]const u8 = null;
+ var it = transfer.responseHeaderIterator();
+ while (it.next()) |hdr| {
+ if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-allow-origin")) {
+ allow_origin = hdr.value;
+ } else if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-allow-credentials")) {
+ allow_credentials = hdr.value;
+ }
+ }
+
+ const allowed = allow_origin orelse return false;
+ if (std.mem.eql(u8, allowed, "*")) {
+ // the wildcard is not valid for credentialed requests
+ return !self._with_credentials;
+ }
+ const origin = self._exec.origin() orelse "null";
+ if (!std.mem.eql(u8, allowed, origin)) {
+ return false;
+ }
+ if (self._with_credentials) {
+ const creds = allow_credentials orelse return false;
+ return std.mem.eql(u8, creds, "true");
+ }
+ return true;
+}
+
+fn httpDataCallback(transfer: *Transfer, data: []const u8) !void {
+ const self: *EventSource = @ptrCast(@alignCast(transfer.req.ctx));
+ try self.parse(data);
+}
+
+fn httpDoneCallback(ctx: *anyopaque) !void {
+ const self: *EventSource = @ptrCast(@alignCast(ctx));
+ self._transfer = null;
+ self.reestablish();
+}
+
+fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
+ const self: *EventSource = @ptrCast(@alignCast(ctx));
+ self._transfer = null;
+ if (self._ready_state == .closed) {
+ // our own doing: close(), a failed connection's .abort, or teardown
+ return;
+ }
+ log.info(.http, "request error", .{
+ .source = "eventsource",
+ .url = self._url,
+ .err = err,
+ });
+ if (err == error.ResponseTooLarge) {
+ // oversized batch, line or event: reconnecting would just download
+ // the same stream again
+ return self.failConnection();
+ }
+ self.reestablish();
+}
+
+fn httpShutdownCallback(ctx: *anyopaque) void {
+ const self: *EventSource = @ptrCast(@alignCast(ctx));
+ self._transfer = null;
+ self._ready_state = .closed;
+ self.deactivate();
+}
+
+fn parse(self: *EventSource, chunk: []const u8) !void {
+ var rest = chunk;
+
+ // a CR ended the previous chunk; its LF partner may lead this one
+ if (self._skip_lf and rest.len > 0) {
+ self._skip_lf = false;
+ if (rest[0] == '\n') {
+ rest = rest[1..];
+ }
+ }
+
+ while (rest.len > 0) {
+ // lines end at CR, LF or CRLF
+ const idx = std.mem.indexOfAny(u8, rest, "\r\n") orelse {
+ // no terminator; hold the partial line for the next chunk
+ return self.bufferLine(rest);
+ };
+ try self.bufferLine(rest[0..idx]);
+
+ if (rest[idx] == '\r') {
+ if (idx + 1 == rest.len) {
+ // chunk ends on this CR; its LF partner may lead the next
+ self._skip_lf = true;
+ rest = rest[idx + 1 ..];
+ } else {
+ rest = rest[idx + @as(usize, if (rest[idx + 1] == '\n') 2 else 1) ..];
+ }
+ } else {
+ rest = rest[idx + 1 ..];
+ }
+ try self.processLine();
+ }
+}
+
+// HttpClient only enforces the max-response per buffered chunk. Now we're
+// accumulating these chunks together.
+fn bufferLine(self: *EventSource, bytes: []const u8) !void {
+ if (self._line_buf.items.len + bytes.len > self._max_response_size) {
+ return error.ResponseTooLarge;
+ }
+ return self._line_buf.appendSlice(self._arena, bytes);
+}
+
+fn processLine(self: *EventSource) !void {
+ var line: []const u8 = self._line_buf.items;
+ defer self._line_buf.clearRetainingCapacity();
+
+ if (!self._bom_checked) {
+ self._bom_checked = true;
+ if (std.mem.startsWith(u8, line, "\xEF\xBB\xBF")) {
+ line = line[3..];
+ }
+ }
+
+ if (line.len == 0) {
+ return self.dispatchPending();
+ }
+ if (line[0] == ':') {
+ // comment
+ return;
+ }
+
+ var field = line;
+ var value: []const u8 = "";
+ if (std.mem.indexOfScalar(u8, line, ':')) |colon| {
+ field = line[0..colon];
+ value = line[colon + 1 ..];
+ if (value.len > 0 and value[0] == ' ') {
+ value = value[1..];
+ }
+ }
+
+ const arena = self._arena;
+ if (std.mem.eql(u8, field, "data")) {
+ const add_len = value.len + 1;
+ if (self._data_buf.items.len + add_len > self._max_response_size) {
+ return error.ResponseTooLarge;
+ }
+ try self._data_buf.ensureUnusedCapacity(arena, add_len);
+ self._data_buf.appendSliceAssumeCapacity(value);
+ self._data_buf.appendAssumeCapacity('\n');
+ }
+
+ if (std.mem.eql(u8, field, "event")) {
+ self._event_type_buf.clearRetainingCapacity();
+ try self._event_type_buf.appendSlice(arena, value);
+ }
+
+ if (std.mem.eql(u8, field, "id")) {
+ if (std.mem.indexOfScalar(u8, value, 0) == null) {
+ self._id_buf.clearRetainingCapacity();
+ try self._id_buf.appendSlice(arena, value);
+ }
+ }
+
+ if (std.mem.eql(u8, field, "retry")) {
+ if (std.fmt.parseInt(u32, value, 10)) |ms| {
+ self._reconnect_ms = ms;
+ } else |_| {}
+ }
+ // unknown fields are ignored
+}
+
+// An empty line completed an event block.
+fn dispatchPending(self: *EventSource) !void {
+ self._last_event_id.clearRetainingCapacity();
+ try self._last_event_id.appendSlice(self._arena, self._id_buf.items);
+
+ const data = self._data_buf.items;
+ if (data.len == 0) {
+ self._event_type_buf.clearRetainingCapacity();
+ return;
+ }
+
+ defer {
+ self._data_buf.clearRetainingCapacity();
+ self._event_type_buf.clearRetainingCapacity();
+ }
+
+ // every data field appended a trailing newline; the last one is dropped
+ const body = data[0 .. data.len - 1];
+ const type_buf = self._event_type_buf.items;
+ const is_message = type_buf.len == 0;
+
+ const exec = self._exec;
+ const target = self.asEventTarget();
+ const handler: ?js.Function.Temp = if (is_message) self._on_message else null;
+
+ if (is_message) {
+ if (!exec.hasDirectListeners(target, "message", handler)) {
+ return;
+ }
+ } else if (!exec.hasDirectListeners(target, type_buf, handler)) {
+ return;
+ }
+
+ const typ: String = if (is_message) comptime .wrap("message") else .wrap(type_buf);
+
+ const event = try MessageEvent.initTrusted(typ, .{
+ .data = .{ .string = body },
+ .origin = self._event_origin,
+ .lastEventId = self._last_event_id.items,
+ }, exec.page);
+ try exec.dispatch(target, event.asEvent(), handler, .{ .context = "EventSource message" });
+}
+
+fn dispatchEvent(self: *EventSource, comptime name: []const u8, handler: ?js.Function.Temp) !void {
+ const exec = self._exec;
+ const target = self.asEventTarget();
+ if (exec.hasDirectListeners(target, name, handler)) {
+ const event = try Event.initTrusted(comptime .wrap(name), .{}, exec.page);
+ try exec.dispatch(target, event, handler, .{ .context = "EventSource " ++ name });
+ }
+}
+
+pub fn getUrl(self: *const EventSource) []const u8 {
+ return self._url;
+}
+
+pub fn getReadyState(self: *const EventSource) u16 {
+ return @intFromEnum(self._ready_state);
+}
+
+pub fn getWithCredentials(self: *const EventSource) bool {
+ return self._with_credentials;
+}
+
+pub fn getOnOpen(self: *const EventSource) ?js.Function.Temp {
+ return self._on_open;
+}
+
+pub fn setOnOpen(self: *EventSource, cb_: ?js.Function) !void {
+ if (self._on_open) |old| {
+ old.release();
+ }
+ self._on_open = if (cb_) |cb| try cb.tempWithThis(self) else null;
+}
+
+pub fn getOnMessage(self: *const EventSource) ?js.Function.Temp {
+ return self._on_message;
+}
+
+pub fn setOnMessage(self: *EventSource, cb_: ?js.Function) !void {
+ if (self._on_message) |old| {
+ old.release();
+ }
+ self._on_message = if (cb_) |cb| try cb.tempWithThis(self) else null;
+}
+
+pub fn getOnError(self: *const EventSource) ?js.Function.Temp {
+ return self._on_error;
+}
+
+pub fn setOnError(self: *EventSource, cb_: ?js.Function) !void {
+ if (self._on_error) |old| {
+ old.release();
+ }
+ self._on_error = if (cb_) |cb| try cb.tempWithThis(self) else null;
+}
+
+pub const JsApi = struct {
+ pub const bridge = js.Bridge(EventSource);
+
+ pub const Meta = struct {
+ pub const name = "EventSource";
+ pub const prototype_chain = bridge.prototypeChain();
+ pub var class_id: bridge.ClassId = undefined;
+ };
+
+ pub const constructor = bridge.constructor(EventSource.init, .{});
+
+ pub const CONNECTING = bridge.property(@intFromEnum(ReadyState.connecting), .{ .template = true });
+ pub const OPEN = bridge.property(@intFromEnum(ReadyState.open), .{ .template = true });
+ pub const CLOSED = bridge.property(@intFromEnum(ReadyState.closed), .{ .template = true });
+
+ pub const url = bridge.accessor(EventSource.getUrl, null, .{});
+ pub const readyState = bridge.accessor(EventSource.getReadyState, null, .{});
+ pub const withCredentials = bridge.accessor(EventSource.getWithCredentials, null, .{});
+
+ pub const onopen = bridge.accessor(EventSource.getOnOpen, EventSource.setOnOpen, .{});
+ pub const onmessage = bridge.accessor(EventSource.getOnMessage, EventSource.setOnMessage, .{});
+ pub const onerror = bridge.accessor(EventSource.getOnError, EventSource.setOnError, .{});
+
+ pub const close = bridge.function(EventSource.close, .{});
+};
+
+const testing = @import("../../../testing.zig");
+test "WebApi: EventSource" {
+ const filter: testing.LogFilter = .init(&.{.http});
+ defer filter.deinit();
+ try testing.htmlRunner("net/eventsource.html", .{});
+}
+
+test "WebApi: EventSource in worker" {
+ const filter: testing.LogFilter = .init(&.{.http});
+ defer filter.deinit();
+ try testing.htmlRunner("net/eventsource_worker.html", .{});
+}
diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig
index 9d8e96ca5..d48bc30ba 100644
--- a/src/cdp/domains/fetch.zig
+++ b/src/cdp/domains/fetch.zig
@@ -204,13 +204,7 @@ pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.
.requestId = &id.toInterceptId(transfer.id),
.frameId = &id.toFrameId(transfer.req.frame_id),
.request = network.RequestWriter.init(transfer),
- .resourceType = switch (transfer.req.resource_type) {
- .script => "Script",
- .xhr => "XHR",
- .document => "Document",
- .fetch => "Fetch",
- .stylesheet => "Stylesheet",
- },
+ .resourceType = transfer.req.resource_type.string(),
.networkId = &id.toRequestId(transfer), // matches the Network REQ-ID
}, .{ .session_id = session_id });
@@ -448,13 +442,7 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificati
.requestId = &id.toInterceptId(transfer.id),
.frameId = &id.toFrameId(request.frame_id),
.request = network.RequestWriter.init(transfer),
- .resourceType = switch (request.resource_type) {
- .script => "Script",
- .xhr => "XHR",
- .document => "Document",
- .fetch => "Fetch",
- .stylesheet => "Stylesheet",
- },
+ .resourceType = request.resource_type.string(),
.authChallenge = .{
.origin = "", // TODO get origin, could be the proxy address for example.
.source = if (challenge.source) |s| (if (s == .server) "Server" else "Proxy") else "",
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 2220d6b94..f3f07553e 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -568,13 +568,15 @@ pub fn _tick(self: *Client, timeout_ms: u32, mode: DrainMode) !void {
// based approach as HTTP connections, and then this can be removed
try self.settleConns();
- if (try self.processMessages() == false) {
- // No messages were processed. We need to wait for I/O. The network
- // layer will wake this up if there's acticity.
+ const processed = try self.processMessages();
+ if (processed == false and self.dispatch_queue.first == null) {
+ // No messages were processed and nothing is waiting for dispatch
+ // We need to wait for I/O.
if (running > 0 or self.cdp_link_active) {
{
self.heartbeat.enterWait();
defer self.heartbeat.exitWait();
+ // The network layer will wake this up if there's acticity.
try self.handles.poll(&.{}, @intCast(timeout_ms));
}
// poll only waits, so we do the perform -> process dance again
@@ -681,6 +683,7 @@ fn dispatchCompleted(self: *Client, mode: DrainMode) void {
continue;
}
self.dispatch_count -= 1;
+ transfer._dispatch_queued = false;
transfer.deliver();
}
}
@@ -819,7 +822,7 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
const cache = self.cache orelse return false;
const req = &transfer.req;
- if (req.method != .GET) {
+ if (req.method != .GET or req.streaming) {
return false;
}
@@ -970,7 +973,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {
if (comptime IS_DEBUG) {
log.debug(.browser, "http cache", .{ .key = transfer._cache_key, .metadata = metadata });
}
- cache.put(metadata, transfer.res.stream_buffer.items) catch |err| {
+ cache.put(metadata, transfer.res.buffer.items) catch |err| {
log.warn(.http, "cache put failed", .{ .err = err });
};
}
@@ -1321,7 +1324,11 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
return true;
}
- if (effective_err == null or effective_err.? == error.RecvError) {
+ // A streaming response that has emitted events is committed: its header
+ // (and data) already reached the consumer, so it can't be transparently
+ // retried as an auth challenge — don't even offer it to CDP, parking it
+ // would strand the already-queued events.
+ if (!transfer.res.stream.started and (effective_err == null or effective_err.? == error.RecvError)) {
transfer.detectAuthChallenge(msg.conn);
}
@@ -1442,7 +1449,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
self.cacheStore(transfer);
transfer._cdp_content_length = transfer.getContentLength() orelse 0;
- try transfer.bufferEvents(transfer.res.stream_buffer.items);
+ try transfer.bufferEvents(transfer.res.buffer.items);
return true;
}
@@ -1528,6 +1535,7 @@ pub const Request = struct {
script,
fetch,
stylesheet,
+ eventsource,
// Allowed Values: Document, Stylesheet, Image, Media, Font, Script,
// TextTrack, XHR, Fetch, Prefetch, EventSource, WebSocket, Manifest,
@@ -1540,6 +1548,7 @@ pub const Request = struct {
.script => "Script",
.fetch => "Fetch",
.stylesheet => "Stylesheet",
+ .eventsource => "EventSource",
};
}
};
@@ -1571,6 +1580,11 @@ pub const Request = struct {
// Set by syncRequest; only used to label the http_requests metric.
sync: bool = false,
+ // Deliver the response body progressively: data_callback can fire multiple
+ // times. Currently, streaming request always bypass the cache as they are only
+ // used for EventSource.
+ streaming: bool = false,
+
// When false, the caller does not guarantee that the body outlives the
// transfer, and thus we'll need to dupe it.
body_outlives_request: bool = false,
@@ -1820,16 +1834,17 @@ pub const Transfer = struct {
// pending queue first, so the node is always free by then).
_node: std.DoublyLinkedList.Node = .{},
- // Buffered response ordered events awaiting dispatch. Today, ArrayList is
- // overkill. _events will be at most len == 4 and almost always len == 4.
- // BUT, when WebSockets move to a dispatch mode AND we re-add streaming,
- // this will become more dynamic.
+ // Buffered response ordered events awaiting dispatch.
_events: std.ArrayList(Event) = .{},
// controls if _queue_node is in client.dispatch_queue (false) or
// client.gated_queue (true)
_gated: bool = false,
+ // True while _queue_node is linked into dispatch_queue/gated_queue.
+ // A streaming transfer can be queued while still .inflight (in the multi)
+ _dispatch_queued: bool = false,
+
_queue_node: std.DoublyLinkedList.Node = .{},
// Mirror this transfer's lifecycle as CDP Network.* events (serve mode).
@@ -1874,10 +1889,11 @@ pub const Transfer = struct {
// for us.
inflight,
- // processOneMessage is running user callbacks. The conn may
- // still be in `_conn` (header/data phase) or have been cleared
- // by the "release ASAP" step before done_callback fires.
- completing,
+ // deliver() is running user callbacks (abort() also latches this
+ // to defer re-entrant teardown). For a buffered transfer no conn
+ // is held; a streaming transfer's conn is still in the multi and
+ // the state returns to .inflight after the batch.
+ delivering,
// External owner is holding the transfer paused. The owner is
// responsible for resuming or terminating it.
@@ -1991,15 +2007,17 @@ pub const Transfer = struct {
self.client.pending_queue.remove(&self._node);
}
- // Same for the dispatch queue: a buffered transfer killed
- // out-of-band (owner teardown, abort) must not be delivered.
- if (self.state == .buffered) {
+ // Same for the dispatch queue: a queued transfer (buffered, or
+ // streaming with a pending batch) killed out-of-band (owner
+ // teardown, abort) must not be delivered.
+ if (self._dispatch_queued) {
if (self._gated) {
self.client.gated_queue.remove(&self._queue_node);
} else {
self.client.dispatch_queue.remove(&self._queue_node);
}
self.client.dispatch_count -= 1;
+ self._dispatch_queued = false;
}
// And for the robots gate: RobotsGate.pending holds a raw *Transfer
@@ -2053,10 +2071,10 @@ pub const Transfer = struct {
pub fn abort(self: *Transfer, err: anyerror) void {
// error_callback can run JS that tears this transfer down again
// (e.g. an XHR abort handler navigates -> abortRequests -> kill).
- // Hold the state at .completing so the re-entrant teardown defers,
+ // Hold the state at .delivering so the re-entrant teardown defers,
// then do the single real teardown against the original state.
const state = self.state;
- self.state = .completing;
+ self.state = .delivering;
self.requestFailed(err);
self.state = state;
self.detachOrDeinit();
@@ -2099,10 +2117,10 @@ pub const Transfer = struct {
// eventually drains the in-flight curl handle.
//
// Two states force deferral:
- // * `.completing` — processOneMessage is currently processing
- // this transfer. It will call `transfer.deinit` itself after the
- // chain returns; deiniting here would double-free. This covers
- // both the with-conn and post-release-ASAP windows.
+ // * `.delivering` — deliver() is running this transfer's callbacks.
+ // It will call `transfer.deinit` itself after the chain returns;
+ // deiniting here would double-free. This covers both the with-conn
+ // and post-release-ASAP windows.
// * `.inflight` while `client.performing` — libcurl could still
// fire callbacks for us. Releasing the arena now would UAF
// from inside curl.
@@ -2112,7 +2130,7 @@ pub const Transfer = struct {
// inline.
fn detachOrDeinit(self: *Transfer) void {
const must_defer = switch (self.state) {
- .completing => true,
+ .delivering => true,
.inflight => self.client.performing,
else => false,
};
@@ -2198,6 +2216,9 @@ pub const Transfer = struct {
// dispatcher owns the transfer's fate; deinit unlinks it if it dies
// out-of-band first.
fn scheduleDispatch(self: *Transfer) void {
+ if (self.state == .delivering) {
+ return;
+ }
if (comptime IS_DEBUG) {
lp.assert(
self.state == .created or self.state == .inflight,
@@ -2206,6 +2227,14 @@ pub const Transfer = struct {
);
}
self.state = .buffered;
+ self.enqueueDispatch();
+ }
+
+ fn enqueueDispatch(self: *Transfer) void {
+ if (self._dispatch_queued) {
+ return;
+ }
+ self._dispatch_queued = true;
self.client.dispatch_queue.append(&self._queue_node);
self.client.dispatch_count += 1;
}
@@ -2218,6 +2247,12 @@ pub const Transfer = struct {
// hops never reach here (they loop back in processOneMessage), so
// http_status reflects the final response only. Internal requests
// (robots.txt) are tracked by the robots_* metrics instead.
+ if (self.res.stream.started) {
+ try self._events.append(self.arena, .done);
+ self.scheduleDispatch();
+ return;
+ }
+
if (!self.req.internal) {
const status = if (self.res.header) |h| h.status else 0;
lp.metrics.http_status.incr(http.statusCategory(status));
@@ -2300,6 +2335,10 @@ pub const Transfer = struct {
// transfer arena. After this, delivery never touches libcurl state —
// the conn can be released and reused before the consumer sees a byte.
fn materializeResponse(self: *Transfer, conn: *http.Connection) !void {
+ if (self.res.stream.started) {
+ // Streaming: already materialized at the first delivered chunk.
+ return;
+ }
const arena = self.arena;
if (self.res.header == null) {
@@ -2363,6 +2402,8 @@ pub const Transfer = struct {
// Per-request timeout override (e.g. XHR timeout)
if (req.timeout_ms > 0) {
try conn.setTimeout(req.timeout_ms);
+ } else if (req.streaming) {
+ try conn.setTimeout(0);
}
// add credentials
@@ -2382,8 +2423,11 @@ pub const Transfer = struct {
// total hops. The rest of the response state is per-attempt.
self._notified_fail = false;
self._tries += 1;
- self.res.stream_buffer.clearRetainingCapacity();
- self.res = .{ .stream_buffer = self.res.stream_buffer };
+ self.res.buffer.clearRetainingCapacity();
+ self.res = .{
+ .buffer = self.res.buffer,
+ .stream = .{ .spare = self.res.stream.spare },
+ };
}
fn buildResponseHeader(self: *Transfer, conn: *const http.Connection) !void {
@@ -2575,20 +2619,41 @@ pub const Transfer = struct {
res.callback_error = error.ResponseTooLarge;
return http.writefunc_error;
}
- res.stream_buffer.ensureTotalCapacity(transfer.arena, cl) catch {};
+ res.buffer.ensureTotalCapacity(transfer.arena, cl) catch {};
}
}
- if (res.skip_body) return @intCast(chunk_len);
+ if (res.skip_body) {
+ return @intCast(chunk_len);
+ }
res.bytes_received += chunk_len;
+
+ const chunk = buffer[0..chunk_len];
+
+ if (transfer.req.streaming) {
+ if (transfer.state == .aborted) {
+ return http.writefunc_error;
+ }
+ // A stream's total is unbounded by design; cap the undelivered
+ // batch instead — that's what actually occupies memory.
+ if (res.buffer.items.len + chunk_len > transfer.client.max_response_size) {
+ res.callback_error = error.ResponseTooLarge;
+ return http.writefunc_error;
+ }
+ transfer.streamChunk(conn, chunk) catch |err| {
+ res.callback_error = err;
+ return http.writefunc_error;
+ };
+ return @intCast(chunk_len);
+ }
+
if (res.bytes_received > transfer.client.max_response_size) {
res.callback_error = error.ResponseTooLarge;
return http.writefunc_error;
}
- const chunk = buffer[0..chunk_len];
- res.stream_buffer.appendSlice(transfer.arena, chunk) catch |err| {
+ res.buffer.appendSlice(transfer.arena, chunk) catch |err| {
res.callback_error = err;
return http.writefunc_error;
};
@@ -2600,6 +2665,34 @@ pub const Transfer = struct {
return @intCast(chunk_len);
}
+ fn streamChunk(self: *Transfer, conn: *http.Connection, chunk: []const u8) !void {
+ const res = &self.res;
+ if (res.stream.started == false) {
+ // we haven't delivered the start/header events yet
+ try self.materializeResponse(conn);
+ try self._events.ensureUnusedCapacity(self.arena, 3);
+ self._events.appendAssumeCapacity(.start);
+ self._events.appendAssumeCapacity(.header);
+ res.stream.started = true;
+ }
+ // append the data to whatever data we already have (but haven't delivered)
+ try res.buffer.appendSlice(self.arena, chunk);
+ if (res.stream.data_queued == false) {
+ res.stream.data_queued = true;
+ try self._events.append(self.arena, .stream_data);
+ }
+
+ switch (self.state) {
+ .inflight => self.enqueueDispatch(),
+ // deliver() is mid-batch on this transfer; its loop consumes
+ // events appended behind its cursor.
+ .delivering => {},
+ else => if (comptime IS_DEBUG) {
+ lp.assert(false, "Transfer.scheduleStreamDispatch", .{ .state = self.state });
+ },
+ }
+ }
+
// Response-view accessors. Callbacks receive the *Transfer once the
// response is materialized, so these are always safe from a callback;
// header/status are null before that point.
@@ -2651,12 +2744,20 @@ pub const Transfer = struct {
return null;
}
- // Only ever called from tick, delivers a completed event to its caller
+ // Only ever called from tick, delivers buffered events to their consumer.
+ // A buffered (completed) transfer gets exactly one deliver() and dies at
+ // the end of it. A streaming transfer gets one deliver() per dispatched
+ // batch and stays inflight between batches, until a terminal event
+ // (done / err) or an abort.
fn deliver(transfer: *Transfer) void {
+ // A streaming batch is delivered while the conn is still inflight;
+ // that state is restored after the batch unless it turned terminal.
+ const was_inflight = transfer.state == .inflight;
+
// Important. If a callback tries to kill the transfer (e.g. abort()),
// this will cause the teardown to be deferred so that it doesn't get
// freed from under us.
- transfer.state = .completing;
+ transfer.state = .delivering;
const req = &transfer.req;
if (transfer._from_cache) {
@@ -2666,24 +2767,19 @@ pub const Transfer = struct {
);
}
- // The current design is that a Transfer will get one and only one
- // deliver(). So at the end of deliver() we can deinit. That also means
- // we can safely iterate transfer._events without risking a mutation to
- // it.
- //
- // When WebSockets and/or buffering is added this will no longer be true
- // and one transfer will be potentially get multiple deliver() calls.
- // When that happens, we need to adjust two things:
- // 1 - It might not be safe to iterate transfer._events like this,
- // something could append to it, making it invalid
- // 2 - Only terminal states (done, err, abort) should call deinit
-
- for (transfer._events.items) |event| {
+ // Index loop, re-reading len and items on every iteration: a
+ // streaming transfer's callback can re-enter the pump (syncRequest)
+ // and append more events — possibly the terminal one — to the batch
+ // being delivered.
+ var terminal = !was_inflight;
+ var i: usize = 0;
+ while (i < transfer._events.items.len) : (i += 1) {
if (transfer.state == .aborted) {
// the state can become aborted as events are processed, check
// it on every iteration.
break;
}
+ const event = transfer._events.items[i];
switch (event) {
.start => {
if (req.start_callback) |cb| {
@@ -2716,7 +2812,31 @@ pub const Transfer = struct {
return transfer.failDelivery(err);
};
},
+ .stream_data => {
+ // Move the batch into stream.spare and deliver it from
+ // there: chunks received during the callback (re-entrant
+ // pump) append to the swapped-in buffer and queue a new
+ // .stream_data that this loop picks up, while the batch
+ // the consumer is reading stays untouched. The batch is
+ // only valid during the callback — it's cleared right
+ // after, ready to be swapped back in.
+ const res = &transfer.res;
+ res.stream.data_queued = false;
+ std.mem.swap(std.ArrayList(u8), &res.buffer, &res.stream.spare);
+ const chunk = res.stream.spare.items;
+ if (transfer._notify_cdp) {
+ req.notification.dispatch(.http_response_data, &.{
+ .data = chunk,
+ .transfer = transfer,
+ });
+ }
+ req.data_callback(transfer, chunk) catch |err| {
+ return transfer.failDelivery(err);
+ };
+ res.stream.spare.clearRetainingCapacity();
+ },
.done => {
+ terminal = true;
if (transfer._notify_cdp) {
req.notification.dispatch(.http_request_done, &.{
.transfer = transfer,
@@ -2728,12 +2848,21 @@ pub const Transfer = struct {
};
},
.err => |err| {
+ terminal = true;
transfer.requestFailed(err);
break;
},
}
}
- transfer.deinit();
+
+ if (transfer.state == .aborted or terminal) {
+ transfer.deinit();
+ return;
+ }
+
+ // Mid-stream batch fully delivered; the conn is still receiving.
+ transfer._events.clearRetainingCapacity();
+ transfer.state = .inflight;
}
};
@@ -2756,11 +2885,28 @@ const Response = struct {
skip_body: bool = false,
first_data_received: bool = false,
- // Buffered response body. Filled by dataCallback, consumed in processMessages.
- stream_buffer: std.ArrayList(u8) = .{},
+ // Response body. Filled by dataCallback, consumed in processMessages.
+ // See Stream.spare to see how this works in streaming mode
+ buffer: std.ArrayList(u8) = .{},
// Error captured in dataCallback to be reported in processMessages.
callback_error: ?anyerror = null,
+
+ // State for streaming. Unused in non-streaming
+ stream: Stream = .{},
+
+ const Stream = struct {
+ // Whether we've queued the start/header events or not
+ started: bool = false,
+
+ // We only queue one .stream_data event at a time, it drains whatever
+ // has accumulated since the last deliver
+ data_queued: bool = false,
+
+ // Along with the main Response.buffer, acts as a double buffer allowing
+ // us to accumulate new data while in a delivery callback.
+ spare: std.ArrayList(u8) = .{},
+ };
};
// A single buffered response event. The payload of `data` has transfer-arena
@@ -2769,6 +2915,7 @@ const Event = union(enum) {
start,
header,
data: []const u8,
+ stream_data, // the payload is carried in res.stream.data_queued
done,
err: anyerror,
};
@@ -2987,7 +3134,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {
test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {
// Regression: the fulfilled response's done_callback runs JS which
// navigates / closes the page, re-entrantly killing the transfer
- // (abortOwner -> kill). deliver() holds the transfer in `.completing`,
+ // (abortOwner -> kill). deliver() holds the transfer in `.delivering`,
// so the teardown defers and there is exactly one free.
var pool = ArenaPool.init(testing.allocator, .{});
defer pool.deinit();
@@ -3300,7 +3447,7 @@ test "HttpClient: fulfillIntercepted delivers a 3xx without a Location as the re
test "HttpClient: abortParked survives an error_callback that tears down the owner" {
// Same re-entrancy hazard as fulfillRequest, but on the abort path
// (failRequest / continueWithAuth-cancel / session teardown). abortParked
- // buffers the failure; deliver() fires it while .completing, so a
+ // buffers the failure; deliver() fires it while .delivering, so a
// re-entrant owner teardown defers to the single deinit instead of
// double-freeing.
var pool = ArenaPool.init(testing.allocator, .{});
@@ -3377,7 +3524,7 @@ test "HttpClient: abort survives an error_callback that tears down the owner" {
// Regression: abort() fired error_callback before detachOrDeinit, so JS
// in that callback that killed the transfer again (navigation ->
// abortRequests -> kill) freed it inline and abort() then ran
- // detachOrDeinit on freed memory. abort holds .completing across
+ // detachOrDeinit on freed memory. abort holds .delivering across
// requestFailed so the re-entrant kill defers to the single teardown.
var pool = ArenaPool.init(testing.allocator, .{});
defer pool.deinit();
@@ -3469,6 +3616,7 @@ test "HttpClient: abort survives an error_callback that tears down the owner" {
transfer.owner = &owner;
transfer.state = .buffered;
+ transfer._dispatch_queued = true;
client.dispatch_queue.append(&transfer._queue_node);
client.dispatch_count += 1;
diff --git a/src/testing.zig b/src/testing.zig
index 47bab7db7..cc3a191da 100644
--- a/src/testing.zig
+++ b/src/testing.zig
@@ -508,6 +508,12 @@ var test_http_server_thread: ?std.Thread = null;
var test_ws_server: ?TestWSServer = null;
var test_ws_server_thread: ?std.Thread = null;
+// Server-side state for the /sse/* endpoints. sse_flag proves progressive
+// delivery (see /sse/streaming); sse_reconnect_hits makes /sse/reconnect
+// serve a different stream on the second connection.
+var sse_flag = std.atomic.Value(bool).init(false);
+var sse_reconnect_hits = std.atomic.Value(usize).init(0);
+
var test_config: Config = undefined;
test "tests:beforeAll" {
@@ -743,6 +749,97 @@ fn testHTTPHandler(req: *std.http.Server.Request) !void {
});
}
+ if (std.mem.eql(u8, path, "/sse/simple")) {
+ // A complete stream: default + custom event types, multi-line data,
+ // an id, and a comment. The connection closes right after; the test
+ // close()s before the (1s) reconnect fires.
+ return req.respond("retry: 1000\ndata: first\n\n: comment\nid: 42\nevent: custom\ndata: a\ndata: b\n\n", .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/event-stream" },
+ },
+ });
+ }
+
+ if (std.mem.eql(u8, path, "/sse/streaming")) {
+ sse_flag.store(false, .release);
+ var send_buffer: [1024]u8 = undefined;
+ var res = try req.respondStreaming(&send_buffer, .{
+ .respond_options = .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/event-stream" },
+ },
+ },
+ });
+ try res.writer.writeAll("data: first\n\n");
+ try res.writer.flush();
+ try res.flush();
+
+ // Proof of progressive delivery: "second" is only sent once the
+ // client has reacted to "first" (by fetching /sse/flag), which it
+ // can only do if "first" was delivered while this response was
+ // still streaming.
+ var waited: usize = 0;
+ while (!sse_flag.load(.acquire) and waited < 5000) : (waited += 10) {
+ std.Thread.sleep(10 * std.time.ns_per_ms);
+ }
+ if (sse_flag.load(.acquire)) {
+ // split mid-line to exercise buffering across chunks
+ try res.writer.writeAll("data: sec");
+ try res.writer.flush();
+ try res.flush();
+ std.Thread.sleep(20 * std.time.ns_per_ms);
+ try res.writer.writeAll("ond\n\n");
+ try res.writer.flush();
+ try res.flush();
+ }
+ return res.end();
+ }
+
+ if (std.mem.eql(u8, path, "/sse/flag")) {
+ sse_flag.store(true, .release);
+ return req.respond("", .{});
+ }
+
+ if (std.mem.eql(u8, path, "/sse/reconnect")) {
+ if (sse_reconnect_hits.fetchAdd(1, .monotonic) == 0) {
+ return req.respond("retry: 10\ndata: one\n\n", .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/event-stream" },
+ },
+ });
+ }
+ return req.respond("data: two\n\n", .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/event-stream" },
+ },
+ });
+ }
+
+ if (std.mem.eql(u8, path, "/sse/long_retry")) {
+ return req.respond("retry: 60000\ndata: x\n\n", .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/event-stream" },
+ },
+ });
+ }
+
+ if (std.mem.eql(u8, path, "/sse/404")) {
+ return req.respond("data: x\n\n", .{
+ .status = .not_found,
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/event-stream" },
+ },
+ });
+ }
+
+ if (std.mem.eql(u8, path, "/sse/badmime")) {
+ return req.respond("data: x\n\n", .{
+ .extra_headers = &.{
+ .{ .name = "Content-Type", .value = "text/plain" },
+ },
+ });
+ }
+
if (std.mem.eql(u8, path, "/styles/visibility.css")) {
// Used by css/external_stylesheet.html — drives the visibility
// cascade through StyleManager via Frame.loadExternalStylesheet