Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
test: specify UTF-16-aware source reading across all repo-source readers
Alex Matthews7b2f3b445d26
diff --git a/test/utf16-source.test.ts b/test/utf16-source.test.tsnew file mode 100644index 00000000..c6dca0c9--- /dev/null+++ b/test/utf16-source.test.ts@@ -0,0 +1,164 @@+/**+ * A1 — UTF-16-aware source reading, routed through every repo-source reader.+ *+ * `util/source.ts`'s `readSourceFile` decodes UTF-16LE (BOM `FF FE`) and treats+ * UTF-16BE (BOM `FE FF`) as unsupported (null — never a silent mojibake read).+ * Every reader that walks real repo source — graph/build.ts, graph/fingerprint.ts,+ * graph/check.ts, context/build.ts, context/check.ts, ask.ts's `--source` span+ * slicer, and search/grep.ts — routes through it, so a UTF-16LE file (the common+ * shape written by Windows tooling generally, not just one language's toolchain)+ * decodes identically everywhere it is read: hash-what-you-parse consistency+ * across build/fingerprint/check/grep/ask, not a fresh mojibake bug per reader.+ *+ * These fixtures deliberately use plain .ts/.py files — no PowerShell grammar on+ * this branch — to prove the routing is general-purpose, not tied to any one+ * source language.+ */+import { test } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync, writeFileSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { readSourceFile } from "../src/util/source.js";+import { buildGraph } from "../src/graph/build.js";+import { checkGraph } from "../src/graph/check.js";+import { probeDrift } from "../src/graph/fingerprint.js";+import { buildContext } from "../src/context/build.js";+import { checkContext } from "../src/context/check.js";+import { grepGraph } from "../src/search/grep.js";+import { ask } from "../src/ask/ask.js";+import { readGraph, wiringPath } from "../src/graph/write.js";+import { contextDirFor } from "../src/context/node-file.js";+import { fakeProviders } from "./helpers.js";++/** BOM + UTF-16LE bytes — the shape Windows tooling in general actually writes. */+function utf16le(text: string): Buffer {+ return Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(text, "utf16le")]);+}++/** BOM + UTF-16BE bytes — unsupported; readSourceFile must return null, never+ * mojibake, for this encoding. */+function utf16be(text: string): Buffer {+ return Buffer.concat([Buffer.from([0xfe, 0xff]), Buffer.from(text, "utf16le").swap16()]);+}++test("readSourceFile: decodes a UTF-16LE BOM, returns null for UTF-16BE, and reads plain UTF-8 unchanged", () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-utf16-unit-"));+ try {+ writeFileSync(join(dir, "le.ts"), utf16le("export const x = 1;\n"));+ writeFileSync(join(dir, "be.ts"), utf16be("export const x = 1;\n"));+ writeFileSync(join(dir, "plain.ts"), "export const x = 1;\n", "utf8");++ assert.equal(readSourceFile(join(dir, "le.ts")), "export const x = 1;\n");+ assert.equal(readSourceFile(join(dir, "be.ts")), null);+ assert.equal(readSourceFile(join(dir, "plain.ts")), "export const x = 1;\n");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("A1 graph/build.ts: a UTF-16LE .ts file is decoded at graph ingest, not mojibake-parsed", async () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-utf16-build-"));+ try {+ writeFileSync(join(dir, "legacy.ts"), utf16le("export function fromLegacy(): void {}\n"));+ const r = await buildGraph(dir);+ assert.deepEqual(r.errors, []);+ const graph = readGraph(wiringPath(join(dir, "graft")))!;+ assert.ok(graph.nodes.some((n) => n.id === "legacy.ts#fromLegacy"), "the UTF-16LE file's function must be extracted");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("A1 graph/build.ts: a UTF-16BE file is an empty entry, not a build error (null → empty-entry posture)", async () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-utf16be-build-"));+ try {+ writeFileSync(join(dir, "legacy.ts"), utf16be("export function fromLegacy(): void {}\n"));+ const r = await buildGraph(dir);+ assert.deepEqual(r.errors, [], "an unsupported encoding is a skip, never an error");+ const graph = readGraph(wiringPath(join(dir, "graft")))!;+ assert.ok(!graph.nodes.some((n) => n.path === "legacy.ts"), "no nodes minted for the undecodable file");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("A1 hash-what-you-parse consistency: build/check/fingerprint agree on a UTF-16LE file", async () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-utf16-consistency-"));+ try {+ writeFileSync(join(dir, "legacy.py"), utf16le("def from_legacy():\n return 42\n"));+ await buildGraph(dir);++ const check = checkGraph(dir);+ assert.equal(check.ok, true, "checkGraph must decode the same way build.ts did — no added/removed/changed");+ assert.deepEqual(check.added, []);+ assert.deepEqual(check.removed, []);+ assert.deepEqual(check.changed, []);++ const drift = probeDrift(join(dir), join(dir, "graft"));+ assert.deepEqual(drift, { changed: [], added: [], removed: [] }, "fingerprint's probe must agree too — no drift");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("A1 context/build.ts + check.ts: a UTF-16LE file's summarizer input decodes cleanly, and the two tiers' content hash stays consistent", async () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-utf16-context-"));+ try {+ writeFileSync(+ join(dir, "legacy.ts"),+ utf16le("// [[Utf16Widget]]\nexport const widget = 1;\n"),+ );+ const r = await buildContext(dir, { model: "fake", ...fakeProviders() });+ assert.deepEqual(r.errors, [], "the file must be readable, not treated as an error");+ // PassthroughSummarizer hands the (post-decode) code straight to+ // BracketSynthesizer, which only matches [[Name]] in CLEAN text — a+ // mojibake decode (NUL bytes between every ASCII char) would never match.+ assert.equal(r.nodes, 1, "the [[Utf16Widget]] marker must survive decoding to be synthesized as a node");++ const check = checkContext(dir);+ assert.equal(check.ok, true, "content hash must agree between build and check for the same UTF-16LE file");+ assert.deepEqual(check.contentDrift, []);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("A1 search/grep.ts: finds a pattern inside a UTF-16LE file", async () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-utf16-grep-"));+ try {+ writeFileSync(+ join(dir, "legacy.ts"),+ utf16le("export function fromLegacy(): void {\n console.log(\"NEEDLE hit\");\n}\n"),+ );+ await buildGraph(dir);+ const graph = readGraph(wiringPath(join(dir, "graft")))!;+ assert.ok(graph.nodes.some((n) => n.id === "legacy.ts#fromLegacy"), "sanity: the file parses (extract already routes through readSourceFile)");++ const r = grepGraph(graph, dir, "NEEDLE");+ assert.equal(r.totalHits, 1, "grep must find the pattern in the UTF-16LE file's decoded text");+ assert.match(r.groups[0]!.hits[0]!.text, /NEEDLE hit/);+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});++test("A1 ask.ts --source: inlines clean (non-garbled) source sliced from a UTF-16LE file", async () => {+ const dir = mkdtempSync(join(tmpdir(), "graft-utf16-ask-"));+ try {+ writeFileSync(+ join(dir, "legacy.ts"),+ utf16le("export function fromLegacyAsk(): number {\n return 42;\n}\n"),+ );+ await buildGraph(dir);++ const r = ask(dir, "fromLegacyAsk", { source: true });+ const hit = r.hits.find((h) => h.title.startsWith("fromLegacyAsk"));+ assert.ok(hit, "expected a hit for the UTF-16LE-defined function");+ assert.ok(hit!.code, "expected inlined source");+ assert.match(hit!.code!, /return 42/, "the sliced span must decode cleanly, not as mojibake");+ assert.ok(!hit!.code!.includes("\u0000"), "no stray NUL bytes from a mis-decoded UTF-16LE read");+ } finally {+ rmSync(dir, { recursive: true, force: true });+ }+});