Skip to content

Lightpanda: the headless browser designed for AI and automation

`URL`: replace `ensureEncoded` with `resolve`

Halil Durak7bf68a484da7
diff --git a/src/browser/URL.zig b/src/browser/URL.zig
index ca5b39c21..27223a882 100644
--- a/src/browser/URL.zig
+++ b/src/browser/URL.zig
@@ -17,7 +17,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
-const idna = @import("../sys/idna.zig");
const U = @import("../sys/url.zig");
@@ -51,117 +50,7 @@ pub fn resolve(
return allocator.dupeZ(u8, href.slice());
}
-/// IDNA pass: converts a non-ASCII host (`räksmörgås.se`) to its punycode form
-/// (`xn--rksmrgs-5wao1o.se`), validates any ASCII punycode (`xn--…`) labels,
-/// and leaves everything else alone.
-fn ensureHostAscii(allocator: Allocator, url: [:0]const u8) ![:0]const u8 {
- const hostname = getHostname(url);
- if (hostname.len == 0 or (!idna.needsAscii(hostname) and !hasAceLabel(hostname))) {
- return url;
- }
-
- const ascii = try idna.toAscii(allocator, hostname);
-
- // hostname is a slice of url, so its start offset is just pointer arithmetic.
- const start = @intFromPtr(hostname.ptr) - @intFromPtr(url.ptr);
- const end = start + hostname.len;
- var buf = try std.ArrayList(u8).initCapacity(allocator, url.len - hostname.len + ascii.len + 1);
- buf.appendSliceAssumeCapacity(url[0..start]);
- buf.appendSliceAssumeCapacity(ascii);
- buf.appendSliceAssumeCapacity(url[end..]);
- buf.appendAssumeCapacity(0);
- return buf.items[0 .. buf.items.len - 1 :0];
-}
-
-/// True if any dot-separated label of `host` begins with the IDNA ACE prefix
-/// "xn--" (case-insensitive). Such labels are punycode: even though they're
-/// pure ASCII, UTS#46 must decode and validate them, so they can't take the
-/// `needsAscii` fast path.
-fn hasAceLabel(host: []const u8) bool {
- var pos: usize = 0;
- while (std.mem.indexOfScalarPos(u8, host, pos, '-')) |i| {
- pos = i + 1;
- if (i < 2 or i + 1 >= host.len or host[i + 1] != '-') {
- continue;
- }
-
- if (!std.ascii.eqlIgnoreCase(host[i - 2 .. i], "xn")) {
- continue;
- }
-
- const label_start = i - 2;
- if (label_start == 0 or host[label_start - 1] == '.') {
- return true;
- }
- }
- return false;
-}
-
-pub fn ensureEncoded(allocator: Allocator, url_in: [:0]const u8, encoding: []const u8) ![:0]const u8 {
- // Resolve any IDN host first; everything below operates on the ASCII form.
- const url = try ensureHostAscii(allocator, url_in);
-
- const scheme_end = std.mem.indexOf(u8, url, "://");
- const authority_start = if (scheme_end) |end| end + 3 else 0;
- const path_start = std.mem.indexOfScalarPos(u8, url, authority_start, '/') orelse return url;
-
- const query_start = std.mem.indexOfScalarPos(u8, url, path_start, '?');
- const fragment_start = std.mem.indexOfScalarPos(u8, url, query_start orelse path_start, '#');
-
- const path_end = query_start orelse fragment_start orelse url.len;
- const query_end = if (query_start) |_| (fragment_start orelse url.len) else path_end;
-
- const path_to_encode = url[path_start..path_end];
- // Path is always UTF-8 percent encoded per URL spec
- const encoded_path = try percentEncodeSegment(allocator, path_to_encode, .path);
-
- // Query string uses document encoding
- const encoded_query = if (query_start) |qs| blk: {
- const query_to_encode = url[qs + 1 .. query_end];
- break :blk try encodeQueryString(allocator, query_to_encode, encoding);
- } else null;
-
- const encoded_fragment = if (fragment_start) |fs| blk: {
- const fragment_to_encode = url[fs + 1 ..];
- break :blk try percentEncodeSegment(allocator, fragment_to_encode, .query);
- } else null;
-
- if (encoded_path.ptr == path_to_encode.ptr and
- (encoded_query == null or encoded_query.?.ptr == url[query_start.? + 1 .. query_end].ptr) and
- (encoded_fragment == null or encoded_fragment.?.ptr == url[fragment_start.? + 1 ..].ptr))
- {
- // nothing has changed
- return url;
- }
-
- var buf = try std.ArrayList(u8).initCapacity(allocator, url.len + 20);
- try buf.appendSlice(allocator, url[0..path_start]);
- try buf.appendSlice(allocator, encoded_path);
- if (encoded_query) |eq| {
- try buf.append(allocator, '?');
- try buf.appendSlice(allocator, eq);
- }
- if (encoded_fragment) |ef| {
- try buf.append(allocator, '#');
- try buf.appendSlice(allocator, ef);
- }
- try buf.append(allocator, 0);
- return buf.items[0 .. buf.items.len - 1 :0];
-}
-
-/// Selects which RFC 3986 / WHATWG URL Standard percent-encode set to apply.
-///
-/// The `path`, `query`, `query_legacy`, `userinfo`, and `fragment` variants
-/// match the corresponding URL spec sets — they assume the input is already
-/// structured (e.g. `key=val&key=val` for `query`) and only encode characters
-/// disallowed in that location.
-///
-/// `component` is stricter: it encodes everything outside the RFC 3986
-/// unreserved set, including sub-delims (`& = + ! * ' ( ) , $ ;`). Use this
-/// when embedding an arbitrary string as a single URI component such as a
-/// query-parameter value, where reserved characters in the input would
-/// otherwise change the URL's structure.
-pub const EncodeSet = enum { path, query, query_legacy, userinfo, fragment, component };
+const EncodeSet = enum { path, query, query_legacy, userinfo, fragment };
pub fn percentEncodeSegment(allocator: Allocator, segment: []const u8, comptime encode_set: EncodeSet) ![]const u8 {
// Check if encoding is needed
@@ -209,52 +98,6 @@ pub fn percentEncodeSegment(allocator: Allocator, segment: []const u8, comptime
return buf.items;
}
-const h5e = @import("parser/html5ever.zig");
-
-/// Encode a query string using the specified encoding.
-/// For UTF-8, this is standard percent encoding.
-/// For legacy encodings, unmappable characters are replaced with NCRs (&#codepoint;).
-fn encodeQueryString(allocator: Allocator, query: []const u8, encoding: []const u8) ![]const u8 {
- // For UTF-8, use standard percent encoding
- if (std.mem.eql(u8, encoding, "UTF-8")) {
- return percentEncodeSegment(allocator, query, .query);
- }
-
- // For legacy encodings, first encode to the target charset with NCR fallback
- const enc_info = h5e.encoding_for_label(encoding.ptr, encoding.len);
- if (!enc_info.isValid()) {
- // Unknown encoding, fall back to UTF-8
- return percentEncodeSegment(allocator, query, .query);
- }
-
- // Calculate max buffer size for encoded output
- const max_encoded_len = h5e.encoding_max_encode_buffer_length(enc_info.handle.?, query.len);
- if (max_encoded_len == 0) {
- return percentEncodeSegment(allocator, query, .query);
- }
-
- const encode_buf = try allocator.alloc(u8, max_encoded_len);
- defer allocator.free(encode_buf);
-
- // Encode UTF-8 to legacy encoding with NCR fallback
- const result = h5e.encoding_encode_with_ncr(
- enc_info.handle.?,
- query.ptr,
- query.len,
- encode_buf.ptr,
- encode_buf.len,
- );
-
- if (!result.isSuccess()) {
- // Encoding failed, fall back to UTF-8
- return percentEncodeSegment(allocator, query, .query);
- }
-
- // Now percent-encode the result using query_legacy to preserve NCRs
- const encoded_bytes = encode_buf[0..result.bytes_written];
- return percentEncodeSegment(allocator, encoded_bytes, .query_legacy);
-}
-
fn shouldPercentEncode(c: u8, comptime encode_set: EncodeSet) bool {
return switch (c) {
// Unreserved characters (RFC 3986)
@@ -276,10 +119,6 @@ fn shouldPercentEncode(c: u8, comptime encode_set: EncodeSet) bool {
};
}
-fn isNullTerminated(comptime value: type) bool {
- return @typeInfo(value).pointer.sentinel_ptr != null;
-}
-
pub fn isCompleteHTTPUrl(url: []const u8) bool {
if (url.len < 3) { // Minimum is "x://"
return false;
@@ -1143,179 +982,6 @@ test "URL: resolve validates ASCII punycode (xn--) labels" {
try testing.expectError(error.TypeError, resolve(testing.arena_allocator, "https://example.com/", "https://xn--a.pt/x", .{}));
}
-test "URL: hasAceLabel" {
- // ACE prefix at a label start (case-insensitive).
- try testing.expectEqual(true, hasAceLabel("xn--a"));
- try testing.expectEqual(true, hasAceLabel("xn--rksmrgs-5wao1o.se"));
- try testing.expectEqual(true, hasAceLabel("a.xn--b.com"));
- try testing.expectEqual(true, hasAceLabel("XN--ab.com"));
- try testing.expectEqual(true, hasAceLabel("foo.example.xn--p1ai"));
-
- // Has '-', but no ACE label.
- try testing.expectEqual(false, hasAceLabel("example.com"));
- try testing.expectEqual(false, hasAceLabel("my-site.com"));
- try testing.expectEqual(false, hasAceLabel("axn--b.com")); // xn-- not at a label start
- try testing.expectEqual(false, hasAceLabel("x-n--a.com")); // not "xn" before the '-'
- try testing.expectEqual(false, hasAceLabel("-.com"));
- try testing.expectEqual(false, hasAceLabel(""));
-}
-
-test "URL: ensureEncoded" {
- defer testing.reset();
-
- const Case = struct {
- url: [:0]const u8,
- expected: [:0]const u8,
- };
-
- const cases = [_]Case{
- .{
- .url = "https://example.com/over 9000!",
- .expected = "https://example.com/over%209000!",
- },
- .{
- .url = "http://example.com/hello world.html",
- .expected = "http://example.com/hello%20world.html",
- },
- .{
- .url = "https://example.com/file[1].html",
- .expected = "https://example.com/file%5B1%5D.html",
- },
- .{
- .url = "https://example.com/file{name}.html",
- .expected = "https://example.com/file%7Bname%7D.html",
- },
- .{
- .url = "https://example.com/page?query=hello world",
- .expected = "https://example.com/page?query=hello%20world",
- },
- .{
- .url = "https://example.com/page?a=1&b=value with spaces",
- .expected = "https://example.com/page?a=1&b=value%20with%20spaces",
- },
- .{
- .url = "https://example.com/page#section one",
- .expected = "https://example.com/page#section%20one",
- },
- .{
- .url = "https://example.com/my path?query=my value#my anchor",
- .expected = "https://example.com/my%20path?query=my%20value#my%20anchor",
- },
- .{
- .url = "https://example.com/already%20encoded",
- .expected = "https://example.com/already%20encoded",
- },
- .{
- .url = "https://example.com/file%5B1%5D.html",
- .expected = "https://example.com/file%5B1%5D.html",
- },
- .{
- .url = "https://example.com/caf%C3%A9",
- .expected = "https://example.com/caf%C3%A9",
- },
- .{
- .url = "https://example.com/page?query=already%20encoded",
- .expected = "https://example.com/page?query=already%20encoded",
- },
- .{
- .url = "https://example.com/page?a=1&b=value%20here",
- .expected = "https://example.com/page?a=1&b=value%20here",
- },
- .{
- .url = "https://example.com/page#section%20one",
- .expected = "https://example.com/page#section%20one",
- },
- .{
- .url = "https://example.com/part%20encoded and not",
- .expected = "https://example.com/part%20encoded%20and%20not",
- },
- .{
- .url = "https://example.com/page?a=encoded%20value&b=not encoded",
- .expected = "https://example.com/page?a=encoded%20value&b=not%20encoded",
- },
- .{
- .url = "https://example.com/my%20path?query=not encoded#encoded%20anchor",
- .expected = "https://example.com/my%20path?query=not%20encoded#encoded%20anchor",
- },
- .{
- .url = "https://example.com/fully%20encoded?query=also%20encoded#and%20this",
- .expected = "https://example.com/fully%20encoded?query=also%20encoded#and%20this",
- },
- .{
- .url = "https://example.com/path-with_under~tilde",
- .expected = "https://example.com/path-with_under~tilde",
- },
- .{
- .url = "https://example.com/sub-delims!$&'()*+,;=",
- .expected = "https://example.com/sub-delims!$&'()*+,;=",
- },
- .{
- .url = "https://example.com",
- .expected = "https://example.com",
- },
- .{
- .url = "https://example.com?query=value",
- .expected = "https://example.com?query=value",
- },
- .{
- .url = "https://example.com/clean/path",
- .expected = "https://example.com/clean/path",
- },
- .{
- .url = "https://example.com/path?clean=query#clean-fragment",
- .expected = "https://example.com/path?clean=query#clean-fragment",
- },
- .{
- .url = "https://example.com/100% complete",
- .expected = "https://example.com/100%25%20complete",
- },
- .{
- .url = "https://example.com/path?value=100% done",
- .expected = "https://example.com/path?value=100%25%20done",
- },
- .{
- .url = "about:blank",
- .expected = "about:blank",
- },
- };
-
- for (cases) |case| {
- const result = try ensureEncoded(testing.arena_allocator, case.url, "UTF-8");
- try testing.expectString(case.expected, result);
- }
-}
-
-test "URL: percentEncodeSegment component passes unreserved chars through" {
- defer testing.reset();
- const r = try percentEncodeSegment(testing.arena_allocator, "abcXYZ012-._~", .component);
- try testing.expectString("abcXYZ012-._~", r);
-}
-
-test "URL: percentEncodeSegment component encodes spaces, sub-delims and reserved chars" {
- defer testing.reset();
- const r = try percentEncodeSegment(testing.arena_allocator, "hello world&q=1", .component);
- try testing.expectString("hello%20world%26q%3D1", r);
-
- const r2 = try percentEncodeSegment(testing.arena_allocator, "a+b!c*d", .component);
- try testing.expectString("a%2Bb%21c%2Ad", r2);
-}
-
-test "URL: percentEncodeSegment component encodes UTF-8 bytes" {
- defer testing.reset();
- const r = try percentEncodeSegment(testing.arena_allocator, "café", .component);
- try testing.expectString("caf%C3%A9", r);
-}
-
-test "URL: percentEncodeSegment component re-encodes literal '%' in raw input" {
- defer testing.reset();
- const r = try percentEncodeSegment(testing.arena_allocator, "100%", .component);
- try testing.expectString("100%25", r);
-
- // Even when followed by hex digits, treat as opaque user data.
- const r2 = try percentEncodeSegment(testing.arena_allocator, "100%2A", .component);
- try testing.expectString("100%252A", r2);
-}
-
test "URL: resolve with encoding" {
defer testing.reset();
diff --git a/src/cdp/domains/page.zig b/src/cdp/domains/page.zig
index 808a2fa70..5302862a6 100644
--- a/src/cdp/domains/page.zig
+++ b/src/cdp/domains/page.zig
@@ -308,7 +308,7 @@ fn navigate(cmd: *CDP.Command) !void {
const session = bc.session;
const frame = bc.mainFrame() orelse return error.FrameNotLoaded;
- const encoded_url = try URL.ensureEncoded(frame.call_arena, params.url, "UTF-8");
+ const encoded_url = try URL.resolve(frame.call_arena, "", params.url, .{});
// Fast path: a freshly-created target whose root frame hasn't navigated
// yet has nothing to preserve across the HTTP round-trip. Skip the
diff --git a/src/cdp/domains/target.zig b/src/cdp/domains/target.zig
index d1543e055..15b524c32 100644
--- a/src/cdp/domains/target.zig
+++ b/src/cdp/domains/target.zig
@@ -222,7 +222,7 @@ fn createTarget(cmd: *CDP.Command) !void {
}
if (!std.mem.eql(u8, "about:blank", params.url)) {
- const encoded_url = try URL.ensureEncoded(frame.call_arena, params.url, "UTF-8");
+ const encoded_url = try URL.resolve(frame.call_arena, "", params.url, .{});
try frame.navigate(
encoded_url,
.{ .reason = .address_bar, .kind = .{ .push = null } },
diff --git a/src/lightpanda.zig b/src/lightpanda.zig
index 39e6aff9d..1685fd3bb 100644
--- a/src/lightpanda.zig
+++ b/src/lightpanda.zig
@@ -104,7 +104,7 @@ pub fn fetch(app: *App, browser: *Browser, url: [:0]const u8, opts: FetchOpts) !
{
const frame = page.frame().?;
// not guaranteed to be valid after navigate
- const encoded_url = try URL.ensureEncoded(frame.call_arena, url, "UTF-8");
+ const encoded_url = try URL.resolve(frame.call_arena, "", url, .{});
_ = try frame.navigate(encoded_url, .{
.reason = .address_bar,
.kind = .{ .push = null },