The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.
refactor(lib): split the gateguard code lexer by token class
Affaan Mustafa87aaef56cbfe
diff --git a/scripts/lib/gateguard-code-lexer.js b/scripts/lib/gateguard-code-lexer.jsindex f89c9e101..c45d7e53b 100644--- a/scripts/lib/gateguard-code-lexer.js+++ b/scripts/lib/gateguard-code-lexer.js@@ -110,141 +110,170 @@ function stringSpecials(quote) {return pattern;}+// --- lexCodeLines state ---+// lx carries the text, spec and partial line. Each step reads from index i and+// returns the index of the last consumed character, or null when the text is+// not lexable and the edit cannot be trivial.++function appendCode(lx, piece) {+ lx.code += piece;+ lx.tail = (lx.tail + piece).slice(-2);+}++function endComment(lx, end) {+ const marker = directiveMarker(lx.text.slice(lx.commentStart, end), lx.spec);+ if (marker) appendCode(lx, marker);+}++function endLine(lx) {+ if (endsWithContinuation(lx.code)) return false;+ lx.lines.push(lx.code);+ lx.code = '';+ lx.tail = '';+ return true;+}++function lexNewline(lx, i) {+ const { text } = lx;+ if (lx.state === 'string') return null;+ if (lx.state === 'line') {+ if (text.slice(lx.commentStart, i).trimEnd().endsWith('\\')) return null;+ endComment(lx, i);+ lx.state = 'code';+ }+ if (lx.state === 'block') {+ lx.lines.push(lx.code);+ lx.code = '';+ lx.tail = '';+ } else if (!endLine(lx)) {+ return null;+ }+ return text[i] === '\r' && text[i + 1] === '\n' ? i + 1 : i;+}++function lexLineComment(lx, i) {+ return nextSpecial(LINE_BREAK, lx.text, i) - 1;+}++function lexBlockComment(lx, i) {+ const { text } = lx;+ const ch = text[i];+ if (ch !== '*' && ch !== '/') return nextSpecial(BLOCK_SPECIALS, text, i + 1) - 1;+ if (ch === '*' && text[i + 1] === '/') {+ endComment(lx, i);+ lx.state = 'code';+ appendCode(lx, ' ');+ return i + 1;+ }+ if (ch === '/' && text[i + 1] === '*') return null;+ return i;+}++function lexString(lx, i) {+ const { text, quote } = lx;+ const ch = text[i];+ if (ch !== '\\' && ch !== quote) {+ const end = nextSpecial(stringSpecials(quote), text, i + 1);+ const run = text.slice(i, end);+ appendCode(lx, run);+ lx.stringBody += run;+ return end - 1;+ }+ appendCode(lx, ch);+ if (ch === '\\') {+ const next = text[i + 1];+ if (next === undefined || next === '\n' || next === '\r') return null;+ appendCode(lx, next);+ return i + 1;+ }+ if (lx.spec.noDollarInStrings && lx.stringBody.includes('$')) return null;+ lx.state = 'code';+ return i;+}++function lexCommentStart(lx, i) {+ const { text, spec } = lx;+ const ch = text[i];+ if (spec.lineComment === '//' && ch === '/' && text[i + 1] === '/') {+ lx.state = 'line';+ lx.commentStart = i + 2;+ return i + 1;+ }+ if (spec.lineComment === '#' && ch === '#') {+ lx.state = 'line';+ lx.commentStart = i + 1;+ return i;+ }+ if (spec.blockComment && ch === '/' && text[i + 1] === '*') {+ lx.state = 'block';+ lx.commentStart = i + 2;+ return i + 1;+ }+ return undefined;+}++function startsJsx(spec, text, i) {+ const ch = text[i];+ const next = text[i + 1];+ if (spec.jsx && ch === '<') return isLetter(next) || next === '/' || next === '>' || next === '!';+ return spec.jsx !== undefined && ch === '-' && next === '-' && text[i + 2] === '>';+}++function lexStringStart(lx, i) {+ const { text, spec } = lx;+ const ch = text[i];+ if (spec.tripleQuotes && text[i + 1] === ch && text[i + 2] === ch) return null;+ if (stringPrefixIsUnsafe(lx.tail, spec)) return null;+ lx.state = 'string';+ lx.quote = ch;+ lx.stringBody = '';+ appendCode(lx, ch);+ return i;+}++function lexCode(lx, i) {+ const { text, spec } = lx;+ const ch = text[i];+ if (!lx.specials.chars.has(ch)) {+ const end = nextSpecial(lx.specials.pattern, text, i + 1);+ appendCode(lx, text.slice(i, end));+ return end - 1;+ }+ const comment = lexCommentStart(lx, i);+ if (comment !== undefined) return comment;+ if (spec.forbiddenCode && spec.forbiddenCode.includes(ch)) return null;+ if (startsJsx(spec, text, i)) return null;+ if (ch === "'" && spec.rustChars) {+ const end = rustCharEnd(text, i);+ appendCode(lx, end === -1 ? ch : text.slice(i, end + 1));+ return end === -1 ? i : end;+ }+ if (spec.quotes.includes(ch)) return lexStringStart(lx, i);+ appendCode(lx, ch);+ return i;+}++const LEX_STEPS = { code: lexCode, line: lexLineComment, block: lexBlockComment, string: lexString };++function finishLex(lx) {+ if (lx.state === 'string' || lx.state === 'block') return null;+ if (lx.state === 'line') {+ if (lx.text.slice(lx.commentStart).trimEnd().endsWith('\\')) return null;+ endComment(lx, lx.text.length);+ }+ return endLine(lx) ? lx.lines : null;+}+function lexCodeLines(text, spec) {- const specials = codeSpecials(spec);- const lines = [];- let code = '';- let state = 'code';- let quote = '';- let stringBody = '';- let commentStart = 0;- let tail = '';- const append = piece => {- code += piece;- tail = (tail + piece).slice(-2);- };- const endComment = end => {- const marker = directiveMarker(text.slice(commentStart, end), spec);- if (marker) append(marker);- };- const endLine = () => {- if (endsWithContinuation(code)) return false;- lines.push(code);- code = '';- tail = '';- return true;- };- if (spec.jsx !== undefined && text.startsWith('#!')) state = 'line';+ const lx = { text, spec, specials: codeSpecials(spec), lines: [], code: '', state: 'code', quote: '', stringBody: '', commentStart: 0, tail: '' };+ if (spec.jsx !== undefined && text.startsWith('#!')) lx.state = 'line';for (let i = 0; i < text.length; i++) {const ch = text[i];- if (ch === '\n' || ch === '\r') {- if (state === 'string') return null;- if (state === 'line') {- if (text.slice(commentStart, i).trimEnd().endsWith('\\')) return null;- endComment(i);- state = 'code';- }- if (state === 'block') {- lines.push(code);- code = '';- tail = '';- } else if (!endLine()) {- return null;- }- if (ch === '\r' && text[i + 1] === '\n') i++;- continue;- }- if (state === 'line') {- i = nextSpecial(LINE_BREAK, text, i) - 1;- continue;- }- if (state === 'block') {- if (ch !== '*' && ch !== '/') {- i = nextSpecial(BLOCK_SPECIALS, text, i + 1) - 1;- continue;- }- if (ch === '*' && text[i + 1] === '/') {- endComment(i);- state = 'code';- append(' ');- i++;- } else if (ch === '/' && text[i + 1] === '*') {- return null;- }- continue;- }- if (state === 'string') {- if (ch !== '\\' && ch !== quote) {- const end = nextSpecial(stringSpecials(quote), text, i + 1);- const run = text.slice(i, end);- append(run);- stringBody += run;- i = end - 1;- continue;- }- append(ch);- if (ch === '\\') {- const next = text[i + 1];- if (next === undefined || next === '\n' || next === '\r') return null;- append(next);- i++;- } else if (ch === quote) {- if (spec.noDollarInStrings && stringBody.includes('$')) return null;- state = 'code';- } else {- stringBody += ch;- }- continue;- }- if (!specials.chars.has(ch)) {- const end = nextSpecial(specials.pattern, text, i + 1);- append(text.slice(i, end));- i = end - 1;- continue;- }- if (spec.lineComment === '//' && ch === '/' && text[i + 1] === '/') {- state = 'line';- commentStart = i + 2;- i++;- continue;- }- if (spec.lineComment === '#' && ch === '#') {- state = 'line';- commentStart = i + 1;- continue;- }- if (spec.blockComment && ch === '/' && text[i + 1] === '*') {- state = 'block';- commentStart = i + 2;- i++;- continue;- }- if (spec.forbiddenCode && spec.forbiddenCode.includes(ch)) return null;- if (spec.jsx && ch === '<' && (isLetter(text[i + 1]) || text[i + 1] === '/' || text[i + 1] === '>' || text[i + 1] === '!')) return null;- if (spec.jsx !== undefined && ch === '-' && text[i + 1] === '-' && text[i + 2] === '>') return null;- if (ch === "'" && spec.rustChars) {- const end = rustCharEnd(text, i);- append(end === -1 ? ch : text.slice(i, end + 1));- if (end !== -1) i = end;- continue;- }- if (spec.quotes.includes(ch)) {- if (spec.tripleQuotes && text[i + 1] === ch && text[i + 2] === ch) return null;- if (stringPrefixIsUnsafe(tail, spec)) return null;- state = 'string';- quote = ch;- stringBody = '';- append(ch);- continue;- }- append(ch);+ const next = ch === '\n' || ch === '\r' ? lexNewline(lx, i) : LEX_STEPS[lx.state](lx, i);+ if (next === null) return null;+ i = next;}- if (state === 'string' || state === 'block') return null;- if (state === 'line') {- if (text.slice(commentStart).trimEnd().endsWith('\\')) return null;- endComment(text.length);- }- return endLine() ? lines : null;+ return finishLex(lx);}function normalizeCodeLine(line, spec) {@@ -364,56 +393,86 @@ function isBlank(ch) {return ch === ' ' || ch === '\t';}+// --- Shell and PowerShell readers ---+// Each reader appends what it consumed to lx.code and returns the index of the+// last consumed character, or -1 when the text is not lexable.++function pushShellLine(lx, i) {+ lx.lines.push(lx.code);+ lx.code = '';+ return lx.text[i] === '\r' && lx.text[i + 1] === '\n' ? i + 1 : i;+}++function readEscapedPair(lx, i) {+ const next = lx.text[i + 1];+ if (next === undefined || next === '\n' || next === '\r') return -1;+ lx.code += lx.text[i] + next;+ return i + 1;+}++function readQuotedSpan(lx, i, end) {+ if (end === -1) return -1;+ lx.code += lx.text.slice(i, end + 1);+ return end;+}++function lineCommentEnd(text, i) {+ while (i + 1 < text.length && text[i + 1] !== '\n' && text[i + 1] !== '\r') i++;+ return i;+}++function readLineComment(lx, i) {+ const end = lineCommentEnd(lx.text, i);+ lx.code += directiveMarker(lx.text.slice(i + 1, end + 1), SHELL_SPEC);+ return end;+}++function shSingleQuoted(lx, i) {+ const { text } = lx;+ if (text[i - 1] === '$') return -1;+ const end = text.indexOf("'", i + 1);+ if (end === -1) return -1;+ const body = text.slice(i, end + 1);+ if (body.includes('\n') || body.includes('\r')) return -1;+ lx.code += body;+ return end;+}++function shDoubleQuoted(lx, i) {+ return readQuotedSpan(lx, i, shDoubleQuoteEnd(lx.text, i));+}++function shHash(lx, i) {+ const starts = hashStartsComment(lx.text, i);+ if (starts === null) return -1;+ if (!starts) {+ lx.code += '#';+ return i;+ }+ return readLineComment(lx, i);+}++const SH_READERS = new Map([['\\', readEscapedPair], ["'", shSingleQuoted], ['"', shDoubleQuoted], ['#', shHash]]);+function shCodeLines(text) {if (text.includes('<<') || text.includes('`') || hasLoneCarriageReturn(text)) return null;- const lines = [];- let code = '';+ const lx = { text, code: '', lines: [] };for (let i = 0; i < text.length; i++) {const ch = text[i];if (ch === '\n' || ch === '\r') {- lines.push(code);- code = '';- if (ch === '\r' && text[i + 1] === '\n') i++;- continue;- }- if (ch === '\\') {- const next = text[i + 1];- if (next === undefined || next === '\n' || next === '\r') return null;- code += ch + next;- i++;+ i = pushShellLine(lx, i);continue;}- if (ch === "'") {- if (text[i - 1] === '$') return null;- const end = text.indexOf("'", i + 1);- if (end === -1) return null;- const body = text.slice(i, end + 1);- if (body.includes('\n') || body.includes('\r')) return null;- code += body;- i = end;+ const reader = SH_READERS.get(ch);+ if (!reader) {+ lx.code += ch;continue;}- if (ch === '"') {- const end = shDoubleQuoteEnd(text, i);- if (end === -1) return null;- code += text.slice(i, end + 1);- i = end;- continue;- }- if (ch === '#') {- const starts = hashStartsComment(text, i);- if (starts === null) return null;- if (starts) {- const start = i + 1;- while (i + 1 < text.length && text[i + 1] !== '\n' && text[i + 1] !== '\r') i++;- code += directiveMarker(text.slice(start, i + 1), SHELL_SPEC);- continue;- }- }- code += ch;+ i = reader(lx, i);+ if (i === -1) return null;}- lines.push(code);- return lines;+ lx.lines.push(lx.code);+ return lx.lines;}const PS_SMART_QUOTES = /[\u2018\u2019\u201a\u201b\u201c\u201d\u201e]/;@@ -442,62 +501,58 @@ function psQuoteEnd(text, i) {return -1;}+function psQuoted(lx, i) {+ return readQuotedSpan(lx, i, psQuoteEnd(lx.text, i));+}++function psBlockStart(lx, i) {+ if (!psHashStartsComment(lx.text, i)) return -1;+ lx.block = true;+ lx.blockStart = i + 2;+ return i + 1;+}++function psHash(lx, i) {+ if (!psHashStartsComment(lx.text, i)) return -1;+ return readLineComment(lx, i);+}++function psBlockComment(lx, i) {+ const { text } = lx;+ if (text[i] === '<' && text[i + 1] === '#') return -1;+ if (text[i] === '#' && text[i + 1] === '>') {+ lx.block = false;+ lx.code += directiveMarker(text.slice(lx.blockStart, i), SHELL_SPEC) || ' ';+ return i + 1;+ }+ return i;+}++function psToken(lx, i) {+ const ch = lx.text[i];+ if (ch === '`') return readEscapedPair(lx, i);+ if (ch === "'" || ch === '"') return psQuoted(lx, i);+ if (ch === '<' && lx.text[i + 1] === '#') return psBlockStart(lx, i);+ if (ch === '#') return psHash(lx, i);+ lx.code += ch;+ return i;+}+function psCodeLines(text) {if (text.includes('@"') || text.includes("@'") || PS_SMART_QUOTES.test(text)) return null;- const lines = [];- let code = '';- let block = false;- let blockStart = 0;+ const lx = { text, code: '', lines: [], block: false, blockStart: 0 };for (let i = 0; i < text.length; i++) {const ch = text[i];if (ch === '\n' || ch === '\r') {- lines.push(code);- code = '';- if (ch === '\r' && text[i + 1] === '\n') i++;- continue;- }- if (block) {- if (ch === '<' && text[i + 1] === '#') return null;- if (ch === '#' && text[i + 1] === '>') {- block = false;- code += directiveMarker(text.slice(blockStart, i), SHELL_SPEC) || ' ';- i++;- }+ i = pushShellLine(lx, i);continue;}- if (ch === '`') {- const next = text[i + 1];- if (next === undefined || next === '\n' || next === '\r') return null;- code += ch + next;- i++;- continue;- }- if (ch === "'" || ch === '"') {- const end = psQuoteEnd(text, i);- if (end === -1) return null;- code += text.slice(i, end + 1);- i = end;- continue;- }- if (ch === '<' && text[i + 1] === '#') {- if (!psHashStartsComment(text, i)) return null;- block = true;- blockStart = i + 2;- i++;- continue;- }- if (ch === '#') {- if (!psHashStartsComment(text, i)) return null;- const start = i + 1;- while (i + 1 < text.length && text[i + 1] !== '\n' && text[i + 1] !== '\r') i++;- code += directiveMarker(text.slice(start, i + 1), SHELL_SPEC);- continue;- }- code += ch;+ i = lx.block ? psBlockComment(lx, i) : psToken(lx, i);+ if (i === -1) return null;}- if (block) return null;- lines.push(code);- return lines;+ if (lx.block) return null;+ lx.lines.push(lx.code);+ return lx.lines;}function batchCodeLines(text) {