Skip to content

Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.

feat: route every repo-source reader through readSourceFile for UTF-16 support

Alex Matthews7fa7a6720457
diff --git a/src/ask/ask.ts b/src/ask/ask.ts
index 35da6e16..e4b7408b 100644
--- a/src/ask/ask.ts
+++ b/src/ask/ask.ts
@@ -32,6 +32,7 @@ import { resolveSymbol } from "../graph/traverse.js";
import type { EdgeV1, GraphV1, NodeV1, Relation } from "../graph/types.js";
import { rankScopesAndFuse } from "./fuse.js";
import { personalizedPageRank } from "./graphrank.js";
+import { readSourceFile } from "../util/source.js";
import { counts, tokenize, type AskIndex, type AskIndexDoc } from "./index-file.js";
export interface AskHit {
@@ -744,7 +745,9 @@ function parseSpan(pointer: string): { path: string; from: number; to: number }
* capped at {@link MAX_SPAN_LINES}. Returns null if the file can't be read. */
function sliceSpan(root: string, path: string, from: number, to: number): string | null {
try {
- const lines = readFileSync(join(root, path), "utf8").split("\n");
+ const source = readSourceFile(join(root, path));
+ if (source === null) return null; // unsupported encoding (e.g. UTF-16BE)
+ const lines = source.split("\n");
const start = Math.max(1, from);
const end = Math.min(lines.length, to);
const slice = lines.slice(start - 1, end);
diff --git a/src/context/build.ts b/src/context/build.ts
index f23d04ae..38db8b2b 100644
--- a/src/context/build.ts
+++ b/src/context/build.ts
@@ -16,6 +16,7 @@ import { join, resolve } from "node:path";
import { walkDir } from "../ingest/fs.js";
import { contentHash } from "../util/id.js";
import { relPosix } from "../util/paths.js";
+import { readSourceFile } from "../util/source.js";
import type { Summarizer } from "../ai/summarize.js";
import type { FileSummary, SynthNode, Synthesizer } from "../ai/synthesize.js";
import {
@@ -123,7 +124,9 @@ export async function buildContext(dir: string, opts: BuildOptions): Promise<Bui
opts.onProgress?.({ phase: "summarize", index: i, total: files.length, file: rel });
let code: string;
try {
- code = readFileSync(file, "utf8");
+ const decoded = readSourceFile(file);
+ if (decoded === null) return undefined; // unsupported encoding (e.g. UTF-16BE) — skip, not an error
+ code = decoded;
} catch (err) {
result.errors.push(`${rel}: ${errMsg(err)}`);
return undefined;
diff --git a/src/context/check.ts b/src/context/check.ts
index a80f7ffa..05d9a041 100644
--- a/src/context/check.ts
+++ b/src/context/check.ts
@@ -12,11 +12,12 @@
* coverage a code file exists that no node was built from (new/uningested)
* index a node's frontmatter disagrees with the manifest (hand-edited)
*/
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync } from "node:fs";
import { resolve } from "node:path";
import { walkDir } from "../ingest/fs.js";
import { contentHash } from "../util/id.js";
import { relPosix } from "../util/paths.js";
+import { readSourceFile } from "../util/source.js";
import { CODE_EXTENSIONS } from "./build.js";
import { contextDirFor, readManifest, readNodes } from "./node-file.js";
@@ -61,7 +62,9 @@ export function checkContext(dir: string, opts: CheckOptions = {}): CheckResult
if (file.startsWith(outDir)) continue;
if (!exts.some((e) => file.toLowerCase().endsWith(e))) continue;
try {
- current.set(relPosix(root, file), contentHash(readFileSync(file, "utf8")));
+ const source = readSourceFile(file);
+ if (source === null) continue; // unsupported encoding (e.g. UTF-16BE) → treat as removed below
+ current.set(relPosix(root, file), contentHash(source));
} catch {
// Unreadable now → treat as removed below (it won't be in `current`).
}
diff --git a/src/graph/build.ts b/src/graph/build.ts
index 336a9b95..07eda91e 100644
--- a/src/graph/build.ts
+++ b/src/graph/build.ts
@@ -20,6 +20,7 @@ import { contextDirFor, ensureGitignored, ensureSearchable } from "../context/no
import { extractFile, languageLabelOf, languageOf, type RawEdge } from "./extract.js";
import { contentHash } from "../util/id.js";
import { relPosix } from "../util/paths.js";
+import { readSourceFile } from "../util/source.js";
import {
emptyExtractCache,
readExtractCache,
@@ -182,9 +183,9 @@ export async function buildGraph(
// may decide whether a *query* bothers rebuilding; it may not decide what the
// rebuild itself looks at. Reading is ~0.05ms/file against the ~4.6ms parse
// this still skips.
- let source: string;
+ let source: string | null;
try {
- source = readFileSync(f.abs, "utf8");
+ source = readSourceFile(f.abs);
} catch (err) {
const message = `${rel}: ${err instanceof Error ? err.message : String(err)}`;
errors.push(message);
@@ -193,6 +194,12 @@ export async function buildGraph(
entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash: "", nodes: [], rawEdges: [], error: message };
return;
}
+ if (source === null) {
+ // Unsupported encoding (UTF-16BE) — a skip, never an error: recorded with
+ // an empty entry so the freshness probe doesn't treat it as new every run.
+ entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash: "", nodes: [], rawEdges: [] };
+ return;
+ }
const hash = contentHash(source);
if (cached && hash === cached.hash) {
diff --git a/src/graph/check.ts b/src/graph/check.ts
index a0bd6d6e..ecd18134 100644
--- a/src/graph/check.ts
+++ b/src/graph/check.ts
@@ -17,13 +17,13 @@
* code. `stale` is a meaning-layer signal the last build already recorded.
* `pending` (never summarized) is not drift — it's a deliberate Tier-1-only build.
*/
-import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { relPosix } from "../util/paths.js";
import { contextDirFor } from "../context/node-file.js";
import { extractFile, languageOf } from "./extract.js";
import { listSourceFiles } from "./build.js";
import { readGraph, wiringPath } from "./write.js";
+import { readSourceFile } from "../util/source.js";
export interface GraphCheckResult {
ok: boolean;
@@ -65,12 +65,13 @@ export function checkGraph(dir: string, opts: GraphCheckOptions = {}): GraphChec
const current = new Map<string, string>(); // id → body_hash
for (const file of listSourceFiles(root, outDir)) {
const lang = languageOf(file)!;
- let source: string;
+ let source: string | null;
try {
- source = readFileSync(file, "utf8");
+ source = readSourceFile(file);
} catch {
continue; // unreadable now → its nodes show up as `removed` below
}
+ if (source === null) continue; // unsupported encoding (e.g. UTF-16BE)
try {
const { nodes } = extractFile(relPosix(root, file), source, lang);
for (const n of nodes) current.set(n.id, n.body_hash);
diff --git a/src/graph/fingerprint.ts b/src/graph/fingerprint.ts
index 440b0798..dd491038 100644
--- a/src/graph/fingerprint.ts
+++ b/src/graph/fingerprint.ts
@@ -18,10 +18,10 @@
* degrade safely: no fingerprint → "unknown, rebuild"; no parse cache → the
* rebuild is just cold.
*/
-import { readFileSync } from "node:fs";
import { join } from "node:path";
import { CACHE_DIR } from "../context/node-file.js";
import { contentHash } from "../util/id.js";
+import { readSourceFile } from "../util/source.js";
import { readJson, writeJsonAtomic } from "../util/state.js";
import { extractorStamp, pruneSidecars, type ExtractEntry } from "./extract-cache.js";
import { listSourceStats } from "./source-files.js";
@@ -158,7 +158,9 @@ export function probeDrift(root: string, outDir: string): Drift | null {
// the only way to learn it's readable again is to try.
let now: string;
try {
- now = contentHash(readFileSync(f.abs, "utf8"));
+ const source = readSourceFile(f.abs);
+ if (source === null) continue; // unsupported encoding (e.g. UTF-16BE)
+ now = contentHash(source);
} catch {
continue; // unreadable right now — leave it to the next probe
}
diff --git a/src/search/grep.ts b/src/search/grep.ts
index 6057b272..52bfe3c7 100644
--- a/src/search/grep.ts
+++ b/src/search/grep.ts
@@ -9,13 +9,13 @@
* human text and wires the CLI command; `mcp/tools.ts` renders the same
* shape for the `graft_find_all` tool.
*/
-import { readFileSync } from "node:fs";
import { join } from "node:path";
import type { GraphV1, NodeV1 } from "../graph/types.js";
import { WALK_RELATIONS } from "../graph/relations.js";
import { assertPrefixIndexed, pathUnderPrefix } from "../graph/scopes.js";
import { normalizePathPrefix } from "../util/paths.js";
import { savingsFor, type Savings } from "../context/savings.js";
+import { readSourceFile } from "../util/source.js";
export interface GrepHit {
line: number;
@@ -160,7 +160,12 @@ export function grepGraph(graph: GraphV1, repoRoot: string, pattern: string, opt
for (const file of fileNodes) {
let text: string;
try {
- text = readFileSync(join(repoRoot, file.path), "utf8");
+ const decoded = readSourceFile(join(repoRoot, file.path));
+ if (decoded === null) {
+ truncatedFiles++; // unsupported encoding (e.g. UTF-16BE) — same posture as unreadable
+ continue;
+ }
+ text = decoded;
} catch {
truncatedFiles++;
continue;
diff --git a/src/util/source.ts b/src/util/source.ts
new file mode 100644
index 00000000..e24d6dde
--- /dev/null
+++ b/src/util/source.ts
@@ -0,0 +1,10 @@
+import { readFileSync } from "node:fs";
+
+/** Read source text, decoding Windows tooling's common UTF-16LE output. UTF-16BE
+ * is rare and unsupported by Node's built-in decoders, so callers silently skip it. */
+export function readSourceFile(path: string): string | null {
+ const bytes = readFileSync(path);
+ if (bytes[0] === 0xfe && bytes[1] === 0xff) return null;
+ if (bytes[0] === 0xff && bytes[1] === 0xfe) return bytes.subarray(2).toString("utf16le");
+ return bytes.toString("utf8");
+}