Lightpanda: the headless browser designed for AI and automation
agent: highlight markdown code blocks as JS
Adrià Arrufat8b1686f6fb00
diff --git a/src/agent/Terminal.zig b/src/agent/Terminal.zigindex 397fdef2b..57f05aa46 100644--- a/src/agent/Terminal.zig+++ b/src/agent/Terminal.zig@@ -25,6 +25,7 @@ const Schema = lp.Schema;const SlashCommand = @import("SlashCommand.zig");const Spinner = @import("Spinner.zig");const md_term = @import("md_term.zig");+const js_highlight = @import("js_highlight.zig");const c = @cImport({@cInclude("isocline.h");});@@ -813,19 +814,9 @@ fn highlightBareToken(henv: ?*c.ic_highlight_env_t, text: []const u8, start: usi} else |_| {}}-/// Returns the index just past the matching closing quote, or `text.len` if-/// unterminated. Does not handle backslash escapes (matches Schema.tokenize).+/// Index just past the matching closing quote, or `text.len` if unterminated.fn scanQuoted(text: []const u8, start: usize) usize {- if (start >= text.len) return start;- const ch = text[start];- const is_triple = start + 2 < text.len and text[start + 1] == ch and text[start + 2] == ch;- if (is_triple) {- const triple_delim = text[start .. start + 3];- const close = std.mem.indexOfPos(u8, text, start + 3, triple_delim) orelse return text.len;- return close + 3;- }- const close = std.mem.indexOfScalarPos(u8, text, start + 1, ch) orelse return text.len;- return close + 1;+ return js_highlight.scanString(text, start).end;}/// Highlight `$LP_*` tokens appearing from `start` onward.@@ -853,95 +844,29 @@ fn highlightDollarVarsIn(henv: ?*c.ic_highlight_env_t, text: []const u8, start:}}-const js_keywords = [_][]const u8{- "function", "async", "await", "yield", "return", "if", "else",- "for", "while", "do", "switch", "case", "break", "continue",- "var", "let", "const", "new", "delete", "typeof", "instanceof",- "in", "of", "void", "this", "super", "class", "extends",- "import", "export", "from", "default", "try", "catch", "finally",- "throw", "true", "false", "null", "undefined", "NaN", "Infinity",-};--// Globals available in the JS-mode page context; highlighted so it's visible-// at the prompt that they're in scope.-const js_globals = [_][]const u8{ "document", "window", "globalThis", "console", "lp" };+/// Paints `js_highlight` spans onto isocline's cell attributes.+const IcSink = struct {+ henv: ?*c.ic_highlight_env_t,-fn isJsKeyword(tok: []const u8) bool {- for (js_keywords) |kw| {- if (std.mem.eql(u8, kw, tok)) return true;- }- return false;-}--fn isJsGlobal(tok: []const u8) bool {- for (js_globals) |g| {- if (std.mem.eql(u8, g, tok)) return true;+ pub fn emit(self: IcSink, start: usize, len: usize, kind: js_highlight.Kind) void {+ const style: []const u8 = switch (kind) {+ .comment => style_comment,+ .string => style_string,+ .variable => style_var,+ .number => style_num,+ .keyword => style_keyword,+ .global => style_jsglobal,+ };+ c.ic_highlight(self.henv, @intCast(start), @intCast(len), style.ptr);}- return false;-}--fn isIdChar(ch: u8) bool {- return std.ascii.isAlphanumeric(ch) or ch == '_' or ch == '$' or ch >= 0x80;-}+};-/// Highlight the buffer as JavaScript: keywords, strings (incl. template-/// literals), numbers, comments, and `$LP_*` env-var refs. Byte offsets are-/// safe (see `highlighterCallback`): every token boundary is an ASCII byte and-/// non-ASCII bytes advance singly without being highlighted.+/// Highlight the buffer as JavaScript. Byte offsets are safe (see+/// `highlighterCallback`): every token boundary is an ASCII byte and non-ASCII+/// bytes advance singly without being highlighted.fn highlightJavaScript(henv: ?*c.ic_highlight_env_t, text: []const u8) void {- var i: usize = 0;- while (i < text.len) {- const ch = text[i];- if (ch == '/' and i + 1 < text.len and (text[i + 1] == '/' or text[i + 1] == '*')) {- const start = i;- if (text[i + 1] == '/') {- i = std.mem.indexOfScalarPos(u8, text, i + 2, '\n') orelse text.len;- } else {- const close = std.mem.indexOfPos(u8, text, i + 2, "*/");- i = if (close) |p| p + 2 else text.len;- }- c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_comment.ptr);- continue;- }- if (ch == '\'' or ch == '"' or ch == '`') {- const start = i;- i = scanQuoted(text, i);- c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_string.ptr);- // `$LP_*` repaints yellow over green: isocline merges per cell, so the later call wins.- highlightDollarVarsIn(henv, text, start, i);- continue;- }- if (ch == '$') {- const start = i;- i += 1;- while (i < text.len and (std.ascii.isAlphanumeric(text[i]) or text[i] == '_')) i += 1;- if (i > start + 1) {- c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_var.ptr);- }- continue;- }- if (std.ascii.isDigit(ch) or (ch == '.' and i + 1 < text.len and std.ascii.isDigit(text[i + 1]))) {- const start = i;- i += 1;- while (i < text.len and (std.ascii.isHex(text[i]) or text[i] == '.' or text[i] == '_' or text[i] == 'x' or text[i] == 'X')) i += 1;- c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_num.ptr);- continue;- }- if (std.ascii.isAlphabetic(ch) or ch == '_') {- const start = i;- i += 1;- while (i < text.len and isIdChar(text[i])) i += 1;- const tok = text[start..i];- if (isJsKeyword(tok)) {- c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_keyword.ptr);- } else if (isJsGlobal(tok) and (start == 0 or text[start - 1] != '.')) {- // `.document` is a property access, not the global- c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_jsglobal.ptr);- }- continue;- }- i += 1;- }+ const sink: IcSink = .{ .henv = henv };+ _ = js_highlight.tokenize(text, .normal, sink);}fn highlightSlashArgs(henv: ?*c.ic_highlight_env_t, text: []const u8, start: usize) void {diff --git a/src/agent/js_highlight.zig b/src/agent/js_highlight.zignew file mode 100644index 000000000..630fb16d8--- /dev/null+++ b/src/agent/js_highlight.zig@@ -0,0 +1,243 @@+// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)+//+// Francis Bouvier <[email protected]>+// Pierre Tachoire <[email protected]>+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++const std = @import("std");++pub const Kind = enum { comment, string, variable, number, keyword, global };++/// Carried between calls because fenced code is tokenized one line at a time:+/// block comments and template literals outlive a line boundary.+pub const State = enum { normal, block_comment, template };++pub const StringSpan = struct { end: usize, closed: bool };++/// Scan the quoted run opening at `text[start]`, including the triple-delimiter+/// form. Escapes are not honored, matching the prompt highlighter's behavior.+pub fn scanString(text: []const u8, start: usize) StringSpan {+ if (start >= text.len) return .{ .end = start, .closed = false };+ const ch = text[start];+ if (start + 2 < text.len and text[start + 1] == ch and text[start + 2] == ch) {+ const delim = text[start .. start + 3];+ const close = std.mem.indexOfPos(u8, text, start + 3, delim) orelse+ return .{ .end = text.len, .closed = false };+ return .{ .end = close + 3, .closed = true };+ }+ const close = std.mem.indexOfScalarPos(u8, text, start + 1, ch) orelse+ return .{ .end = text.len, .closed = false };+ return .{ .end = close + 1, .closed = true };+}++/// Tokenize `text` as JavaScript starting from `state`, reporting spans to+/// `sink.emit(start, len, kind)` and returning the state at end of input.+/// Spans arrive in order and never overlap — the gaps between them are plain+/// text — so an emitter that writes sequentially (ANSI) works as well as one+/// that paints cells (isocline).+pub fn tokenize(text: []const u8, state: State, sink: anytype) State {+ var i: usize = 0;++ switch (state) {+ .block_comment => {+ const close = std.mem.indexOfPos(u8, text, 0, "*/");+ i = if (close) |p| p + 2 else text.len;+ if (i > 0) sink.emit(0, i, .comment);+ if (close == null) return .block_comment;+ },+ .template => {+ const close = std.mem.indexOfScalarPos(u8, text, 0, '`');+ i = if (close) |p| p + 1 else text.len;+ emitString(text, 0, i, sink);+ if (close == null) return .template;+ },+ .normal => {},+ }++ while (i < text.len) {+ const ch = text[i];+ if (ch == '/' and i + 1 < text.len and (text[i + 1] == '/' or text[i + 1] == '*')) {+ const start = i;+ if (text[i + 1] == '/') {+ i = std.mem.indexOfScalarPos(u8, text, i + 2, '\n') orelse text.len;+ sink.emit(start, i - start, .comment);+ continue;+ }+ const close = std.mem.indexOfPos(u8, text, i + 2, "*/");+ i = if (close) |p| p + 2 else text.len;+ sink.emit(start, i - start, .comment);+ if (close == null) return .block_comment;+ continue;+ }+ if (ch == '\'' or ch == '"' or ch == '`') {+ const span = scanString(text, i);+ emitString(text, i, span.end, sink);+ i = span.end;+ // Only a template literal may legally continue on the next line.+ if (!span.closed and ch == '`') return .template;+ continue;+ }+ if (ch == '$') {+ const start = i;+ i += 1;+ while (i < text.len and (std.ascii.isAlphanumeric(text[i]) or text[i] == '_')) i += 1;+ if (i > start + 1) sink.emit(start, i - start, .variable);+ continue;+ }+ if (std.ascii.isDigit(ch) or (ch == '.' and i + 1 < text.len and std.ascii.isDigit(text[i + 1]))) {+ const start = i;+ i += 1;+ while (i < text.len and (std.ascii.isHex(text[i]) or text[i] == '.' or text[i] == '_' or text[i] == 'x' or text[i] == 'X')) i += 1;+ sink.emit(start, i - start, .number);+ continue;+ }+ if (std.ascii.isAlphabetic(ch) or ch == '_') {+ const start = i;+ i += 1;+ while (i < text.len and isIdChar(text[i])) i += 1;+ const tok = text[start..i];+ if (isKeyword(tok)) {+ sink.emit(start, i - start, .keyword);+ } else if (isGlobal(tok) and (start == 0 or text[start - 1] != '.')) {+ // `.document` is a property access, not the global+ sink.emit(start, i - start, .global);+ }+ continue;+ }+ i += 1;+ }+ return .normal;+}++/// Emit `text[start..end]` as string spans split around any `$name` refs, so+/// the two never overlap. The prompt highlighter used to paint the ref over the+/// string and lean on isocline's last-write-wins per cell; splitting keeps the+/// same result while staying safe for a sequential writer.+fn emitString(text: []const u8, start: usize, end: usize, sink: anytype) void {+ var seg = start;+ var i = start;+ while (i < end) {+ if (text[i] != '$') {+ i += 1;+ continue;+ }+ const tok = i;+ i += 1;+ while (i < end and (std.ascii.isAlphanumeric(text[i]) or text[i] == '_')) i += 1;+ if (i > tok + 1) {+ if (tok > seg) sink.emit(seg, tok - seg, .string);+ sink.emit(tok, i - tok, .variable);+ seg = i;+ }+ }+ if (end > seg) sink.emit(seg, end - seg, .string);+}++const keywords = [_][]const u8{+ "function", "async", "await", "yield", "return", "if", "else",+ "for", "while", "do", "switch", "case", "break", "continue",+ "var", "let", "const", "new", "delete", "typeof", "instanceof",+ "in", "of", "void", "this", "super", "class", "extends",+ "import", "export", "from", "default", "try", "catch", "finally",+ "throw", "true", "false", "null", "undefined", "NaN", "Infinity",+};++// Globals available in the JS-mode page context; highlighted so it's visible+// at the prompt that they're in scope.+const globals = [_][]const u8{ "document", "window", "globalThis", "console", "lp" };++pub fn isKeyword(tok: []const u8) bool {+ for (keywords) |kw| {+ if (std.mem.eql(u8, kw, tok)) return true;+ }+ return false;+}++pub fn isGlobal(tok: []const u8) bool {+ for (globals) |g| {+ if (std.mem.eql(u8, g, tok)) return true;+ }+ return false;+}++pub fn isIdChar(ch: u8) bool {+ return std.ascii.isAlphanumeric(ch) or ch == '_' or ch == '$' or ch >= 0x80;+}++const testing = std.testing;++/// Records spans as `kind:text` so tests read as the tokenization, not offsets.+const TestSink = struct {+ text: []const u8,+ buf: std.ArrayListUnmanaged(u8) = .empty,+ last_end: usize = 0,+ overlapped: bool = false,++ fn emit(self: *TestSink, start: usize, len: usize, kind: Kind) void {+ if (start < self.last_end) self.overlapped = true;+ self.last_end = start + len;+ self.buf.writer(testing.allocator).print("{s}:{s} ", .{+ @tagName(kind), self.text[start..][0..len],+ }) catch unreachable;+ }+};++fn expectTokens(expected: []const u8, src: []const u8) !void {+ var sink: TestSink = .{ .text = src };+ defer sink.buf.deinit(testing.allocator);+ _ = tokenize(src, .normal, &sink);+ try testing.expect(!sink.overlapped);+ try testing.expectEqualStrings(expected, std.mem.trimRight(u8, sink.buf.items, " "));+}++test "js_highlight: keywords, globals, numbers" {+ try expectTokens("keyword:const number:1", "const x = 1");+ try expectTokens("global:document", "document.body");+ // A property access is not the global.+ try expectTokens("", "el.document");+}++test "js_highlight: comments and strings" {+ try expectTokens("comment:// hi", "// hi");+ try expectTokens("comment:/* hi */ keyword:const", "/* hi */ const");+ try expectTokens("string:'hi'", "'hi'");+}++test "js_highlight: $refs inside strings do not overlap" {+ try expectTokens("string:'a variable:$LP_KEY string:'", "'a $LP_KEY'");+ try expectTokens("variable:$LP_KEY", "$LP_KEY");+}++test "js_highlight: block comment spans lines" {+ var sink: TestSink = .{ .text = "/* open" };+ defer sink.buf.deinit(testing.allocator);+ try testing.expectEqual(State.block_comment, tokenize("/* open", .normal, &sink));++ var sink2: TestSink = .{ .text = "still */ const" };+ defer sink2.buf.deinit(testing.allocator);+ try testing.expectEqual(State.normal, tokenize("still */ const", .block_comment, &sink2));+ try testing.expectEqualStrings("comment:still */ keyword:const", std.mem.trimRight(u8, sink2.buf.items, " "));+}++test "js_highlight: template literal spans lines" {+ var sink: TestSink = .{ .text = "`<div>" };+ defer sink.buf.deinit(testing.allocator);+ try testing.expectEqual(State.template, tokenize("`<div>", .normal, &sink));++ var sink2: TestSink = .{ .text = "</div>` + x" };+ defer sink2.buf.deinit(testing.allocator);+ try testing.expectEqual(State.normal, tokenize("</div>` + x", .template, &sink2));+ try testing.expectEqualStrings("string:</div>`", std.mem.trimRight(u8, sink2.buf.items, " "));+}diff --git a/src/agent/md_term.zig b/src/agent/md_term.zigindex 864d891d2..d9b697a53 100644--- a/src/agent/md_term.zig+++ b/src/agent/md_term.zig@@ -17,6 +17,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.const std = @import("std");+const js_highlight = @import("js_highlight.zig");pub const ansi = struct {pub const reset = "\x1b[0m";@@ -29,6 +30,8 @@ pub const ansi = struct {pub const green = "\x1b[32m";pub const yellow = "\x1b[33m";pub const red = "\x1b[31m";+ pub const blue = "\x1b[34m";+ pub const magenta = "\x1b[35m";pub const clear_eol = "\x1b[K";pub const clear_line = "\x1b[2K";};@@ -36,12 +39,14 @@ pub const ansi = struct {/// Render markdown `src` as ANSI-styled terminal output to `w`.pub fn render(w: *std.Io.Writer, src: []const u8) !void {var in_fence = false;+ var js: js_highlight.State = .normal;var wrote_any = false;var it = std.mem.splitScalar(u8, src, '\n');while (it.next()) |line| {// Drop the ``` delimiter entirely: no line, no separator.if (isFenceDelimiter(line)) {in_fence = !in_fence;+ js = .normal;continue;}if (wrote_any) try w.writeByte('\n');@@ -52,7 +57,7 @@ pub fn render(w: *std.Io.Writer, src: []const u8) !void {continue;};}- try renderLine(w, line, in_fence);+ try renderLine(w, line, if (in_fence) &js else null);}}@@ -273,6 +278,8 @@ fn isTableSeparator(line: []const u8) bool {pub const Stream = struct {len: usize = 0,in_fence: bool = false,+ /// Fenced-code lexer state, carried across lines and chunks.+ js_state: js_highlight.State = .normal,/// A line that outgrew `buf` passes through unrendered to its newline.raw: bool = false,/// `held`: one pipe row buffered, pending the separator that confirms a@@ -343,11 +350,13 @@ pub const Stream = struct {}const partial = self.buf[0..self.len];const in_fence = self.in_fence;+ var js = self.js_state;self.len = 0;self.raw = false;self.in_fence = false;+ self.js_state = .normal;if (partial.len == 0 or isFenceDelimiter(partial)) return;- try renderLine(w, partial, in_fence);+ try renderLine(w, partial, if (in_fence) &js else null);}fn emitLine(self: *Stream, w: *std.Io.Writer, text: []const u8) std.Io.Writer.Error!void {@@ -355,13 +364,14 @@ pub const Stream = struct {.text => {if (isFenceDelimiter(text)) {self.in_fence = !self.in_fence;+ self.js_state = .normal;return;}if (!self.in_fence and isTableRow(text) and self.appendTableLine(text)) {self.mode = .held;return;}- try renderLine(w, text, self.in_fence);+ try renderLine(w, text, if (self.in_fence) &self.js_state else null);try w.writeByte('\n');},.held => {@@ -379,7 +389,7 @@ pub const Stream = struct {if (isTableRow(text)) {if (self.appendTableLine(text)) return;try self.flushTableUnaligned(w);- try renderLine(w, text, false);+ try renderLine(w, text, null);try w.writeByte('\n');return;}@@ -391,7 +401,7 @@ pub const Stream = struct {/// No separator followed, so the held row wasn't a table header.fn releaseHeldRow(self: *Stream, w: *std.Io.Writer) std.Io.Writer.Error!void {- try renderLine(w, self.tableText(), false);+ try renderLine(w, self.tableText(), null);try w.writeByte('\n');self.resetTable();}@@ -409,7 +419,7 @@ pub const Stream = struct {try w.writeAll(clear_placeholder);var it = std.mem.splitScalar(u8, self.tableText(), '\n');while (it.next()) |line| {- try renderLine(w, line, false);+ try renderLine(w, line, null);try w.writeByte('\n');}self.resetTable();@@ -437,9 +447,10 @@ pub const Stream = struct {}};-fn renderLine(w: *std.Io.Writer, line: []const u8, in_fence: bool) !void {- if (in_fence) {- try styled(w, line, ansi.cyan);+/// `js` carries the fenced-code lexer state across lines; null outside a fence.+fn renderLine(w: *std.Io.Writer, line: []const u8, js: ?*js_highlight.State) !void {+ if (js) |state| {+ state.* = try renderCodeLine(w, line, state.*);return;}@@ -571,6 +582,53 @@ fn span(w: *std.Io.Writer, inner: []const u8, style: []const u8, active: ?*consttry Style.applyOpt(active, w);}+/// Writes `js_highlight` spans as ANSI, filling the gaps between them with+/// unstyled text. `emit` cannot fail, so a write error is stashed and returned+/// by `finish`.+const JsSink = struct {+ w: *std.Io.Writer,+ text: []const u8,+ last: usize = 0,+ err: ?std.Io.Writer.Error = null,++ fn color(kind: js_highlight.Kind) []const u8 {+ return switch (kind) {+ .comment => ansi.dim ++ ansi.italic,+ .string => ansi.green,+ .variable => ansi.yellow ++ ansi.bold,+ .number => ansi.magenta,+ .keyword => ansi.blue ++ ansi.bold,+ .global => ansi.cyan,+ };+ }++ pub fn emit(self: *JsSink, start: usize, len: usize, kind: js_highlight.Kind) void {+ self.write(start, len, kind) catch |err| {+ self.err = self.err orelse err;+ };+ }++ fn write(self: *JsSink, start: usize, len: usize, kind: js_highlight.Kind) !void {+ if (start > self.last) try self.w.writeAll(self.text[self.last..start]);+ try styled(self.w, self.text[start..][0..len], color(kind));+ self.last = start + len;+ }++ fn finish(self: *JsSink) !void {+ if (self.err) |err| return err;+ if (self.last < self.text.len) try self.w.writeAll(self.text[self.last..]);+ }+};++/// Syntax-highlight one line of fenced code as JavaScript — the only language+/// this agent emits in practice. Returns the lexer state for the next line.+fn renderCodeLine(w: *std.Io.Writer, line: []const u8, state: js_highlight.State) !js_highlight.State {+ var sink: JsSink = .{ .w = w, .text = line };+ const next = js_highlight.tokenize(line, state, &sink);+ try sink.finish();+ return next;+}+fn styled(w: *std.Io.Writer, inner: []const u8, style: []const u8) !void {try w.writeAll(style);try w.writeAll(inner);@@ -661,8 +719,20 @@ test "md_term: nested inline styles" {);}-test "md_term: fenced code block" {- try expectRender("\x1b[36mlet x = 1;\x1b[0m", "```\nlet x = 1;\n```");+test "md_term: fenced code block is highlighted as JavaScript" {+ try expectRender(+ "\x1b[34m\x1b[1mlet\x1b[0m x = \x1b[35m1\x1b[0m;",+ "```\nlet x = 1;\n```",+ );+ // Untokenized text passes through unstyled.+ try expectRender("plain", "```\nplain\n```");+}++test "md_term: fenced template literal spans lines" {+ try expectRender(+ "\x1b[32m`<div>\x1b[0m\n\x1b[32m</div>`\x1b[0m",+ "```\n`<div>\n</div>`\n```",+ );}test "md_term: link" {@@ -798,7 +868,7 @@ test "md_term: stream fence state spans chunks" {try s.feed(&aw.writer, "`\nafter\n");try s.close(&aw.writer);try testing.expectEqualStrings(- "\x1b[36mcode\x1b[0m\nafter\n",+ "code\nafter\n",aw.written(),);}