Turbocharge Claude Code, Cursor, Codex, Gemini & every coding agent: faster, cheaper, with contextual understanding specific to your codebase.
TypeScript ai-agentsanthropicclaude-codeclicode-graphcodexcontext-engineeringcursordeveloper-toolsgeminiknowledge-graphllmmcpmcp-serveropen-sourceopenaitree-sitter
feat: route every repo-source reader through readSourceFile for UTF-16 support
Alex Matthews7fa7a6720457
diff --git a/src/ask/ask.ts b/src/ask/ask.tsindex 35da6e16..e4b7408b 100644--- a/src/ask/ask.ts+++ b/src/ask/ask.ts@@ -32,6 +32,7 @@ import { resolveSymbol } from "../graph/traverse.js";import type { EdgeV1, GraphV1, NodeV1, Relation } from "../graph/types.js";import { rankScopesAndFuse } from "./fuse.js";import { personalizedPageRank } from "./graphrank.js";+import { readSourceFile } from "../util/source.js";import { counts, tokenize, type AskIndex, type AskIndexDoc } from "./index-file.js";export interface AskHit {@@ -744,7 +745,9 @@ function parseSpan(pointer: string): { path: string; from: number; to: number }* capped at {@link MAX_SPAN_LINES}. Returns null if the file can't be read. */function sliceSpan(root: string, path: string, from: number, to: number): string | null {try {- const lines = readFileSync(join(root, path), "utf8").split("\n");+ const source = readSourceFile(join(root, path));+ if (source === null) return null; // unsupported encoding (e.g. UTF-16BE)+ const lines = source.split("\n");const start = Math.max(1, from);const end = Math.min(lines.length, to);const slice = lines.slice(start - 1, end);diff --git a/src/context/build.ts b/src/context/build.tsindex f23d04ae..38db8b2b 100644--- a/src/context/build.ts+++ b/src/context/build.ts@@ -16,6 +16,7 @@ import { join, resolve } from "node:path";import { walkDir } from "../ingest/fs.js";import { contentHash } from "../util/id.js";import { relPosix } from "../util/paths.js";+import { readSourceFile } from "../util/source.js";import type { Summarizer } from "../ai/summarize.js";import type { FileSummary, SynthNode, Synthesizer } from "../ai/synthesize.js";import {@@ -123,7 +124,9 @@ export async function buildContext(dir: string, opts: BuildOptions): Promise<Buiopts.onProgress?.({ phase: "summarize", index: i, total: files.length, file: rel });let code: string;try {- code = readFileSync(file, "utf8");+ const decoded = readSourceFile(file);+ if (decoded === null) return undefined; // unsupported encoding (e.g. UTF-16BE) — skip, not an error+ code = decoded;} catch (err) {result.errors.push(`${rel}: ${errMsg(err)}`);return undefined;diff --git a/src/context/check.ts b/src/context/check.tsindex a80f7ffa..05d9a041 100644--- a/src/context/check.ts+++ b/src/context/check.ts@@ -12,11 +12,12 @@* coverage a code file exists that no node was built from (new/uningested)* index a node's frontmatter disagrees with the manifest (hand-edited)*/-import { existsSync, readFileSync } from "node:fs";+import { existsSync } from "node:fs";import { resolve } from "node:path";import { walkDir } from "../ingest/fs.js";import { contentHash } from "../util/id.js";import { relPosix } from "../util/paths.js";+import { readSourceFile } from "../util/source.js";import { CODE_EXTENSIONS } from "./build.js";import { contextDirFor, readManifest, readNodes } from "./node-file.js";@@ -61,7 +62,9 @@ export function checkContext(dir: string, opts: CheckOptions = {}): CheckResultif (file.startsWith(outDir)) continue;if (!exts.some((e) => file.toLowerCase().endsWith(e))) continue;try {- current.set(relPosix(root, file), contentHash(readFileSync(file, "utf8")));+ const source = readSourceFile(file);+ if (source === null) continue; // unsupported encoding (e.g. UTF-16BE) → treat as removed below+ current.set(relPosix(root, file), contentHash(source));} catch {// Unreadable now → treat as removed below (it won't be in `current`).}diff --git a/src/graph/build.ts b/src/graph/build.tsindex 336a9b95..07eda91e 100644--- a/src/graph/build.ts+++ b/src/graph/build.ts@@ -20,6 +20,7 @@ import { contextDirFor, ensureGitignored, ensureSearchable } from "../context/noimport { extractFile, languageLabelOf, languageOf, type RawEdge } from "./extract.js";import { contentHash } from "../util/id.js";import { relPosix } from "../util/paths.js";+import { readSourceFile } from "../util/source.js";import {emptyExtractCache,readExtractCache,@@ -182,9 +183,9 @@ export async function buildGraph(// may decide whether a *query* bothers rebuilding; it may not decide what the// rebuild itself looks at. Reading is ~0.05ms/file against the ~4.6ms parse// this still skips.- let source: string;+ let source: string | null;try {- source = readFileSync(f.abs, "utf8");+ source = readSourceFile(f.abs);} catch (err) {const message = `${rel}: ${err instanceof Error ? err.message : String(err)}`;errors.push(message);@@ -193,6 +194,12 @@ export async function buildGraph(entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash: "", nodes: [], rawEdges: [], error: message };return;}+ if (source === null) {+ // Unsupported encoding (UTF-16BE) — a skip, never an error: recorded with+ // an empty entry so the freshness probe doesn't treat it as new every run.+ entries[rel] = { size: f.size, mtimeMs: f.mtimeMs, hash: "", nodes: [], rawEdges: [] };+ return;+ }const hash = contentHash(source);if (cached && hash === cached.hash) {diff --git a/src/graph/check.ts b/src/graph/check.tsindex a0bd6d6e..ecd18134 100644--- a/src/graph/check.ts+++ b/src/graph/check.ts@@ -17,13 +17,13 @@* code. `stale` is a meaning-layer signal the last build already recorded.* `pending` (never summarized) is not drift — it's a deliberate Tier-1-only build.*/-import { readFileSync } from "node:fs";import { resolve } from "node:path";import { relPosix } from "../util/paths.js";import { contextDirFor } from "../context/node-file.js";import { extractFile, languageOf } from "./extract.js";import { listSourceFiles } from "./build.js";import { readGraph, wiringPath } from "./write.js";+import { readSourceFile } from "../util/source.js";export interface GraphCheckResult {ok: boolean;@@ -65,12 +65,13 @@ export function checkGraph(dir: string, opts: GraphCheckOptions = {}): GraphChecconst current = new Map<string, string>(); // id → body_hashfor (const file of listSourceFiles(root, outDir)) {const lang = languageOf(file)!;- let source: string;+ let source: string | null;try {- source = readFileSync(file, "utf8");+ source = readSourceFile(file);} catch {continue; // unreadable now → its nodes show up as `removed` below}+ if (source === null) continue; // unsupported encoding (e.g. UTF-16BE)try {const { nodes } = extractFile(relPosix(root, file), source, lang);for (const n of nodes) current.set(n.id, n.body_hash);diff --git a/src/graph/fingerprint.ts b/src/graph/fingerprint.tsindex 440b0798..dd491038 100644--- a/src/graph/fingerprint.ts+++ b/src/graph/fingerprint.ts@@ -18,10 +18,10 @@* degrade safely: no fingerprint → "unknown, rebuild"; no parse cache → the* rebuild is just cold.*/-import { readFileSync } from "node:fs";import { join } from "node:path";import { CACHE_DIR } from "../context/node-file.js";import { contentHash } from "../util/id.js";+import { readSourceFile } from "../util/source.js";import { readJson, writeJsonAtomic } from "../util/state.js";import { extractorStamp, pruneSidecars, type ExtractEntry } from "./extract-cache.js";import { listSourceStats } from "./source-files.js";@@ -158,7 +158,9 @@ export function probeDrift(root: string, outDir: string): Drift | null {// the only way to learn it's readable again is to try.let now: string;try {- now = contentHash(readFileSync(f.abs, "utf8"));+ const source = readSourceFile(f.abs);+ if (source === null) continue; // unsupported encoding (e.g. UTF-16BE)+ now = contentHash(source);} catch {continue; // unreadable right now — leave it to the next probe}diff --git a/src/search/grep.ts b/src/search/grep.tsindex 6057b272..52bfe3c7 100644--- a/src/search/grep.ts+++ b/src/search/grep.ts@@ -9,13 +9,13 @@* human text and wires the CLI command; `mcp/tools.ts` renders the same* shape for the `graft_find_all` tool.*/-import { readFileSync } from "node:fs";import { join } from "node:path";import type { GraphV1, NodeV1 } from "../graph/types.js";import { WALK_RELATIONS } from "../graph/relations.js";import { assertPrefixIndexed, pathUnderPrefix } from "../graph/scopes.js";import { normalizePathPrefix } from "../util/paths.js";import { savingsFor, type Savings } from "../context/savings.js";+import { readSourceFile } from "../util/source.js";export interface GrepHit {line: number;@@ -160,7 +160,12 @@ export function grepGraph(graph: GraphV1, repoRoot: string, pattern: string, optfor (const file of fileNodes) {let text: string;try {- text = readFileSync(join(repoRoot, file.path), "utf8");+ const decoded = readSourceFile(join(repoRoot, file.path));+ if (decoded === null) {+ truncatedFiles++; // unsupported encoding (e.g. UTF-16BE) — same posture as unreadable+ continue;+ }+ text = decoded;} catch {truncatedFiles++;continue;diff --git a/src/util/source.ts b/src/util/source.tsnew file mode 100644index 00000000..e24d6dde--- /dev/null+++ b/src/util/source.ts@@ -0,0 +1,10 @@+import { readFileSync } from "node:fs";++/** Read source text, decoding Windows tooling's common UTF-16LE output. UTF-16BE+ * is rare and unsupported by Node's built-in decoders, so callers silently skip it. */+export function readSourceFile(path: string): string | null {+ const bytes = readFileSync(path);+ if (bytes[0] === 0xfe && bytes[1] === 0xff) return null;+ if (bytes[0] === 0xff && bytes[1] === 0xfe) return bytes.subarray(2).toString("utf16le");+ return bytes.toString("utf8");+}