Lightpanda: the headless browser designed for AI and automation
`URL`: replace `ensureEncoded` with `resolve`
Halil Durak7bf68a484da7
diff --git a/src/browser/URL.zig b/src/browser/URL.zigindex ca5b39c21..27223a882 100644--- a/src/browser/URL.zig+++ b/src/browser/URL.zig@@ -17,7 +17,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.const std = @import("std");-const idna = @import("../sys/idna.zig");const U = @import("../sys/url.zig");@@ -51,117 +50,7 @@ pub fn resolve(return allocator.dupeZ(u8, href.slice());}-/// IDNA pass: converts a non-ASCII host (`räksmörgås.se`) to its punycode form-/// (`xn--rksmrgs-5wao1o.se`), validates any ASCII punycode (`xn--…`) labels,-/// and leaves everything else alone.-fn ensureHostAscii(allocator: Allocator, url: [:0]const u8) ![:0]const u8 {- const hostname = getHostname(url);- if (hostname.len == 0 or (!idna.needsAscii(hostname) and !hasAceLabel(hostname))) {- return url;- }-- const ascii = try idna.toAscii(allocator, hostname);-- // hostname is a slice of url, so its start offset is just pointer arithmetic.- const start = @intFromPtr(hostname.ptr) - @intFromPtr(url.ptr);- const end = start + hostname.len;- var buf = try std.ArrayList(u8).initCapacity(allocator, url.len - hostname.len + ascii.len + 1);- buf.appendSliceAssumeCapacity(url[0..start]);- buf.appendSliceAssumeCapacity(ascii);- buf.appendSliceAssumeCapacity(url[end..]);- buf.appendAssumeCapacity(0);- return buf.items[0 .. buf.items.len - 1 :0];-}--/// True if any dot-separated label of `host` begins with the IDNA ACE prefix-/// "xn--" (case-insensitive). Such labels are punycode: even though they're-/// pure ASCII, UTS#46 must decode and validate them, so they can't take the-/// `needsAscii` fast path.-fn hasAceLabel(host: []const u8) bool {- var pos: usize = 0;- while (std.mem.indexOfScalarPos(u8, host, pos, '-')) |i| {- pos = i + 1;- if (i < 2 or i + 1 >= host.len or host[i + 1] != '-') {- continue;- }-- if (!std.ascii.eqlIgnoreCase(host[i - 2 .. i], "xn")) {- continue;- }-- const label_start = i - 2;- if (label_start == 0 or host[label_start - 1] == '.') {- return true;- }- }- return false;-}--pub fn ensureEncoded(allocator: Allocator, url_in: [:0]const u8, encoding: []const u8) ![:0]const u8 {- // Resolve any IDN host first; everything below operates on the ASCII form.- const url = try ensureHostAscii(allocator, url_in);-- const scheme_end = std.mem.indexOf(u8, url, "://");- const authority_start = if (scheme_end) |end| end + 3 else 0;- const path_start = std.mem.indexOfScalarPos(u8, url, authority_start, '/') orelse return url;-- const query_start = std.mem.indexOfScalarPos(u8, url, path_start, '?');- const fragment_start = std.mem.indexOfScalarPos(u8, url, query_start orelse path_start, '#');-- const path_end = query_start orelse fragment_start orelse url.len;- const query_end = if (query_start) |_| (fragment_start orelse url.len) else path_end;-- const path_to_encode = url[path_start..path_end];- // Path is always UTF-8 percent encoded per URL spec- const encoded_path = try percentEncodeSegment(allocator, path_to_encode, .path);-- // Query string uses document encoding- const encoded_query = if (query_start) |qs| blk: {- const query_to_encode = url[qs + 1 .. query_end];- break :blk try encodeQueryString(allocator, query_to_encode, encoding);- } else null;-- const encoded_fragment = if (fragment_start) |fs| blk: {- const fragment_to_encode = url[fs + 1 ..];- break :blk try percentEncodeSegment(allocator, fragment_to_encode, .query);- } else null;-- if (encoded_path.ptr == path_to_encode.ptr and- (encoded_query == null or encoded_query.?.ptr == url[query_start.? + 1 .. query_end].ptr) and- (encoded_fragment == null or encoded_fragment.?.ptr == url[fragment_start.? + 1 ..].ptr))- {- // nothing has changed- return url;- }-- var buf = try std.ArrayList(u8).initCapacity(allocator, url.len + 20);- try buf.appendSlice(allocator, url[0..path_start]);- try buf.appendSlice(allocator, encoded_path);- if (encoded_query) |eq| {- try buf.append(allocator, '?');- try buf.appendSlice(allocator, eq);- }- if (encoded_fragment) |ef| {- try buf.append(allocator, '#');- try buf.appendSlice(allocator, ef);- }- try buf.append(allocator, 0);- return buf.items[0 .. buf.items.len - 1 :0];-}--/// Selects which RFC 3986 / WHATWG URL Standard percent-encode set to apply.-///-/// The `path`, `query`, `query_legacy`, `userinfo`, and `fragment` variants-/// match the corresponding URL spec sets — they assume the input is already-/// structured (e.g. `key=val&key=val` for `query`) and only encode characters-/// disallowed in that location.-///-/// `component` is stricter: it encodes everything outside the RFC 3986-/// unreserved set, including sub-delims (`& = + ! * ' ( ) , $ ;`). Use this-/// when embedding an arbitrary string as a single URI component such as a-/// query-parameter value, where reserved characters in the input would-/// otherwise change the URL's structure.-pub const EncodeSet = enum { path, query, query_legacy, userinfo, fragment, component };+const EncodeSet = enum { path, query, query_legacy, userinfo, fragment };pub fn percentEncodeSegment(allocator: Allocator, segment: []const u8, comptime encode_set: EncodeSet) ![]const u8 {// Check if encoding is needed@@ -209,52 +98,6 @@ pub fn percentEncodeSegment(allocator: Allocator, segment: []const u8, comptimereturn buf.items;}-const h5e = @import("parser/html5ever.zig");--/// Encode a query string using the specified encoding.-/// For UTF-8, this is standard percent encoding.-/// For legacy encodings, unmappable characters are replaced with NCRs (&#codepoint;).-fn encodeQueryString(allocator: Allocator, query: []const u8, encoding: []const u8) ![]const u8 {- // For UTF-8, use standard percent encoding- if (std.mem.eql(u8, encoding, "UTF-8")) {- return percentEncodeSegment(allocator, query, .query);- }-- // For legacy encodings, first encode to the target charset with NCR fallback- const enc_info = h5e.encoding_for_label(encoding.ptr, encoding.len);- if (!enc_info.isValid()) {- // Unknown encoding, fall back to UTF-8- return percentEncodeSegment(allocator, query, .query);- }-- // Calculate max buffer size for encoded output- const max_encoded_len = h5e.encoding_max_encode_buffer_length(enc_info.handle.?, query.len);- if (max_encoded_len == 0) {- return percentEncodeSegment(allocator, query, .query);- }-- const encode_buf = try allocator.alloc(u8, max_encoded_len);- defer allocator.free(encode_buf);-- // Encode UTF-8 to legacy encoding with NCR fallback- const result = h5e.encoding_encode_with_ncr(- enc_info.handle.?,- query.ptr,- query.len,- encode_buf.ptr,- encode_buf.len,- );-- if (!result.isSuccess()) {- // Encoding failed, fall back to UTF-8- return percentEncodeSegment(allocator, query, .query);- }-- // Now percent-encode the result using query_legacy to preserve NCRs- const encoded_bytes = encode_buf[0..result.bytes_written];- return percentEncodeSegment(allocator, encoded_bytes, .query_legacy);-}-fn shouldPercentEncode(c: u8, comptime encode_set: EncodeSet) bool {return switch (c) {// Unreserved characters (RFC 3986)@@ -276,10 +119,6 @@ fn shouldPercentEncode(c: u8, comptime encode_set: EncodeSet) bool {};}-fn isNullTerminated(comptime value: type) bool {- return @typeInfo(value).pointer.sentinel_ptr != null;-}-pub fn isCompleteHTTPUrl(url: []const u8) bool {if (url.len < 3) { // Minimum is "x://"return false;@@ -1143,179 +982,6 @@ test "URL: resolve validates ASCII punycode (xn--) labels" {try testing.expectError(error.TypeError, resolve(testing.arena_allocator, "https://example.com/", "https://xn--a.pt/x", .{}));}-test "URL: hasAceLabel" {- // ACE prefix at a label start (case-insensitive).- try testing.expectEqual(true, hasAceLabel("xn--a"));- try testing.expectEqual(true, hasAceLabel("xn--rksmrgs-5wao1o.se"));- try testing.expectEqual(true, hasAceLabel("a.xn--b.com"));- try testing.expectEqual(true, hasAceLabel("XN--ab.com"));- try testing.expectEqual(true, hasAceLabel("foo.example.xn--p1ai"));-- // Has '-', but no ACE label.- try testing.expectEqual(false, hasAceLabel("example.com"));- try testing.expectEqual(false, hasAceLabel("my-site.com"));- try testing.expectEqual(false, hasAceLabel("axn--b.com")); // xn-- not at a label start- try testing.expectEqual(false, hasAceLabel("x-n--a.com")); // not "xn" before the '-'- try testing.expectEqual(false, hasAceLabel("-.com"));- try testing.expectEqual(false, hasAceLabel(""));-}--test "URL: ensureEncoded" {- defer testing.reset();-- const Case = struct {- url: [:0]const u8,- expected: [:0]const u8,- };-- const cases = [_]Case{- .{- .url = "https://example.com/over 9000!",- .expected = "https://example.com/over%209000!",- },- .{- .url = "http://example.com/hello world.html",- .expected = "http://example.com/hello%20world.html",- },- .{- .url = "https://example.com/file[1].html",- .expected = "https://example.com/file%5B1%5D.html",- },- .{- .url = "https://example.com/file{name}.html",- .expected = "https://example.com/file%7Bname%7D.html",- },- .{- .url = "https://example.com/page?query=hello world",- .expected = "https://example.com/page?query=hello%20world",- },- .{- .url = "https://example.com/page?a=1&b=value with spaces",- .expected = "https://example.com/page?a=1&b=value%20with%20spaces",- },- .{- .url = "https://example.com/page#section one",- .expected = "https://example.com/page#section%20one",- },- .{- .url = "https://example.com/my path?query=my value#my anchor",- .expected = "https://example.com/my%20path?query=my%20value#my%20anchor",- },- .{- .url = "https://example.com/already%20encoded",- .expected = "https://example.com/already%20encoded",- },- .{- .url = "https://example.com/file%5B1%5D.html",- .expected = "https://example.com/file%5B1%5D.html",- },- .{- .url = "https://example.com/caf%C3%A9",- .expected = "https://example.com/caf%C3%A9",- },- .{- .url = "https://example.com/page?query=already%20encoded",- .expected = "https://example.com/page?query=already%20encoded",- },- .{- .url = "https://example.com/page?a=1&b=value%20here",- .expected = "https://example.com/page?a=1&b=value%20here",- },- .{- .url = "https://example.com/page#section%20one",- .expected = "https://example.com/page#section%20one",- },- .{- .url = "https://example.com/part%20encoded and not",- .expected = "https://example.com/part%20encoded%20and%20not",- },- .{- .url = "https://example.com/page?a=encoded%20value&b=not encoded",- .expected = "https://example.com/page?a=encoded%20value&b=not%20encoded",- },- .{- .url = "https://example.com/my%20path?query=not encoded#encoded%20anchor",- .expected = "https://example.com/my%20path?query=not%20encoded#encoded%20anchor",- },- .{- .url = "https://example.com/fully%20encoded?query=also%20encoded#and%20this",- .expected = "https://example.com/fully%20encoded?query=also%20encoded#and%20this",- },- .{- .url = "https://example.com/path-with_under~tilde",- .expected = "https://example.com/path-with_under~tilde",- },- .{- .url = "https://example.com/sub-delims!$&'()*+,;=",- .expected = "https://example.com/sub-delims!$&'()*+,;=",- },- .{- .url = "https://example.com",- .expected = "https://example.com",- },- .{- .url = "https://example.com?query=value",- .expected = "https://example.com?query=value",- },- .{- .url = "https://example.com/clean/path",- .expected = "https://example.com/clean/path",- },- .{- .url = "https://example.com/path?clean=query#clean-fragment",- .expected = "https://example.com/path?clean=query#clean-fragment",- },- .{- .url = "https://example.com/100% complete",- .expected = "https://example.com/100%25%20complete",- },- .{- .url = "https://example.com/path?value=100% done",- .expected = "https://example.com/path?value=100%25%20done",- },- .{- .url = "about:blank",- .expected = "about:blank",- },- };-- for (cases) |case| {- const result = try ensureEncoded(testing.arena_allocator, case.url, "UTF-8");- try testing.expectString(case.expected, result);- }-}--test "URL: percentEncodeSegment component passes unreserved chars through" {- defer testing.reset();- const r = try percentEncodeSegment(testing.arena_allocator, "abcXYZ012-._~", .component);- try testing.expectString("abcXYZ012-._~", r);-}--test "URL: percentEncodeSegment component encodes spaces, sub-delims and reserved chars" {- defer testing.reset();- const r = try percentEncodeSegment(testing.arena_allocator, "hello world&q=1", .component);- try testing.expectString("hello%20world%26q%3D1", r);-- const r2 = try percentEncodeSegment(testing.arena_allocator, "a+b!c*d", .component);- try testing.expectString("a%2Bb%21c%2Ad", r2);-}--test "URL: percentEncodeSegment component encodes UTF-8 bytes" {- defer testing.reset();- const r = try percentEncodeSegment(testing.arena_allocator, "café", .component);- try testing.expectString("caf%C3%A9", r);-}--test "URL: percentEncodeSegment component re-encodes literal '%' in raw input" {- defer testing.reset();- const r = try percentEncodeSegment(testing.arena_allocator, "100%", .component);- try testing.expectString("100%25", r);-- // Even when followed by hex digits, treat as opaque user data.- const r2 = try percentEncodeSegment(testing.arena_allocator, "100%2A", .component);- try testing.expectString("100%252A", r2);-}-test "URL: resolve with encoding" {defer testing.reset();diff --git a/src/cdp/domains/page.zig b/src/cdp/domains/page.zigindex 808a2fa70..5302862a6 100644--- a/src/cdp/domains/page.zig+++ b/src/cdp/domains/page.zig@@ -308,7 +308,7 @@ fn navigate(cmd: *CDP.Command) !void {const session = bc.session;const frame = bc.mainFrame() orelse return error.FrameNotLoaded;- const encoded_url = try URL.ensureEncoded(frame.call_arena, params.url, "UTF-8");+ const encoded_url = try URL.resolve(frame.call_arena, "", params.url, .{});// Fast path: a freshly-created target whose root frame hasn't navigated// yet has nothing to preserve across the HTTP round-trip. Skip thediff --git a/src/cdp/domains/target.zig b/src/cdp/domains/target.zigindex d1543e055..15b524c32 100644--- a/src/cdp/domains/target.zig+++ b/src/cdp/domains/target.zig@@ -222,7 +222,7 @@ fn createTarget(cmd: *CDP.Command) !void {}if (!std.mem.eql(u8, "about:blank", params.url)) {- const encoded_url = try URL.ensureEncoded(frame.call_arena, params.url, "UTF-8");+ const encoded_url = try URL.resolve(frame.call_arena, "", params.url, .{});try frame.navigate(encoded_url,.{ .reason = .address_bar, .kind = .{ .push = null } },diff --git a/src/lightpanda.zig b/src/lightpanda.zigindex 39e6aff9d..1685fd3bb 100644--- a/src/lightpanda.zig+++ b/src/lightpanda.zig@@ -104,7 +104,7 @@ pub fn fetch(app: *App, browser: *Browser, url: [:0]const u8, opts: FetchOpts) !{const frame = page.frame().?;// not guaranteed to be valid after navigate- const encoded_url = try URL.ensureEncoded(frame.call_arena, url, "UTF-8");+ const encoded_url = try URL.resolve(frame.call_arena, "", url, .{});_ = try frame.navigate(encoded_url, .{.reason = .address_bar,.kind = .{ .push = null },