Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

fix(mcp): decode transcript lines only once their newline has arrived

Jack Rudenkob9c238fff564
diff --git a/packages/cli/src/pane/transcript-follower.test.ts b/packages/cli/src/pane/transcript-follower.test.ts
index 9ef93c4a..dcc76978 100644
--- a/packages/cli/src/pane/transcript-follower.test.ts
+++ b/packages/cli/src/pane/transcript-follower.test.ts
@@ -632,3 +632,46 @@ describe("tailing", () => {
expect(whole.turnDurationAfterLast).not.toBeNull();
});
});
+
+describe("byte-safe tail", () => {
+ test("a poll that splits a multi-byte character decodes the record exactly once, offsets in bytes", async () => {
+ const { appendFileSync, mkdtempSync, rmSync, writeFileSync } = await import("node:fs");
+ const { tmpdir } = await import("node:os");
+ const { Tail, TranscriptFollower } = await import("./transcript-follower.js");
+ const dir = mkdtempSync(join(tmpdir(), "pane-tail-utf8-"));
+ cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
+ const path = join(dir, "t.jsonl");
+ writeFileSync(path, "");
+ const w = "Reply with exactly PEAR and nothing else.";
+ const answer = "PEAR — 東京 ✓ 🍐";
+ // the real capture, with the answer text templated to carry multi-byte characters
+ const lines = transcriptRecords("pear-hookless").map((r) => {
+ if (r.type !== "assistant") return JSON.stringify(r);
+ const c = structuredClone(r) as Rec;
+ for (const b of c.message.content) if (b.type === "text") b.text = answer;
+ return JSON.stringify(c);
+ });
+ const bytes = Buffer.from(`${lines.join("\n")}\n`, "utf8");
+ const at = bytes.indexOf(Buffer.from("東", "utf8"));
+ expect(at).toBeGreaterThan(0);
+ const f = new TranscriptFollower(path, join(dir, "subagents"));
+ const tail = new Tail(path);
+ f.openTurn({ index: 1, offset: f.size(), witness: text(w) });
+ appendFileSync(path, bytes.subarray(0, at + 1)); // inside the 3-byte 東
+ f.poll();
+ const first = tail.read();
+ appendFileSync(path, bytes.subarray(at + 1));
+ f.poll();
+ const second = tail.read();
+ expect(cur(f.view()).assistantText).toEqual([answer]);
+ expect(cur(f.view()).turnDurationAfterLast).not.toBeNull();
+ // every line comes back intact, at the byte offset it starts at
+ const got = [...first, ...second];
+ expect(got.map((l) => l.line)).toEqual(lines);
+ let pos = 0;
+ for (const [i, l] of got.entries()) {
+ expect(l.offset).toBe(pos);
+ pos += Buffer.byteLength(lines[i] as string) + 1;
+ }
+ });
+});
diff --git a/packages/cli/src/pane/transcript-follower.ts b/packages/cli/src/pane/transcript-follower.ts
index 94646680..5baf08ea 100644
--- a/packages/cli/src/pane/transcript-follower.ts
+++ b/packages/cli/src/pane/transcript-follower.ts
@@ -580,9 +580,14 @@ export function turnView(t: TurnState): TurnView {
/* ───────────────────────────── file tailing ───────────────────────────── */
-class Tail {
+/**
+ * Partial-line-safe tail. The remainder after the last newline is kept as BYTES and a
+ * line is decoded only once it is complete, so a poll that lands inside a multi-byte
+ * character can never decode either half as U+FFFD; offsets are byte positions.
+ */
+export class Tail {
offset = 0;
- private buf = "";
+ private rest: Buffer = Buffer.alloc(0);
constructor(readonly path: string) {}
/** Read appended bytes; return complete lines with the byte offset each starts at. */
@@ -595,22 +600,23 @@ class Tail {
}
try {
const size = fstatSync(fd).size;
- const startOfBuf = this.offset - Buffer.byteLength(this.buf);
if (size <= this.offset) return [];
const chunk = Buffer.alloc(size - this.offset);
- readSync(fd, chunk, 0, chunk.length, this.offset);
- this.offset = size;
- this.buf += chunk.toString("utf8");
+ const got = readSync(fd, chunk, 0, chunk.length, this.offset);
+ this.offset += got;
+ const fresh = chunk.subarray(0, got);
+ const buf = this.rest.length ? Buffer.concat([this.rest, fresh]) : fresh;
+ let pos = this.offset - buf.length;
const out: Array<{ offset: number; line: string }> = [];
- let pos = startOfBuf;
- let nl = this.buf.indexOf("\n");
+ let start = 0;
+ let nl = buf.indexOf(0x0a, start);
while (nl >= 0) {
- const line = this.buf.slice(0, nl);
- out.push({ offset: pos, line });
- pos += Buffer.byteLength(line) + 1;
- this.buf = this.buf.slice(nl + 1);
- nl = this.buf.indexOf("\n");
+ out.push({ offset: pos, line: buf.toString("utf8", start, nl) });
+ pos += nl + 1 - start;
+ start = nl + 1;
+ nl = buf.indexOf(0x0a, start);
}
+ this.rest = Buffer.from(buf.subarray(start));
return out;
} finally {
closeSync(fd);