Skip to content

Open-source SEO, GEO, and marketing skills for AI agents.

fix(transcript): dedup shadow + OpenClaw double-render; one-shot final flush

Yuting Zhong473f3e5d4ead
diff --git a/src/app/api/agents/[agent]/threads/[thread]/live/route.ts b/src/app/api/agents/[agent]/threads/[thread]/live/route.ts
index 0fac2808..34cbd940 100644
--- a/src/app/api/agents/[agent]/threads/[thread]/live/route.ts
+++ b/src/app/api/agents/[agent]/threads/[thread]/live/route.ts
@@ -1,79 +1,68 @@
+import { watch } from "node:fs";
import { NextResponse } from "next/server";
import { getActiveProject } from "@/server/active-project";
import { resolveAgentBySlug } from "@/server/agent-meta";
import { getProject } from "@/server/db/projects";
-import { GatewayClient } from "@/server/openclaw/gateway-client";
+import { rawEntryToEvents } from "@/server/openclaw/transcript-tail";
import {
- buildPendingSessionKey,
- findSessionBySessionId,
-} from "@/server/openclaw/sessions";
-import {
- rawEntryToEvents,
- type RawEntry,
-} from "@/server/openclaw/transcript-tail";
+ readShadowFromOffset,
+ shadowTranscriptPath,
+} from "@/server/openclaw/shadow-transcript";
export const dynamic = "force-dynamic";
export const runtime = "nodejs";
+const DEBUG = process.env.NOTFAIR_LIVE_BRIDGE_DEBUG !== "0";
+function log(...args: unknown[]): void {
+ if (DEBUG) console.log("[live-bridge]", ...args);
+}
+
/**
- * SSE bridge that re-attaches to an in-progress OpenClaw session and
- * forwards transcript events live. Bypasses the JSONL-flush latency:
- * OpenClaw buffers session.jsonl until session.ended, so polling
- * /api/agents/.../threads/.../transcript sees nothing mid-turn. This
- * endpoint instead subscribes via `sessions.messages.subscribe` over
- * the WS gateway and forwards every `session.message` event as SSE.
+ * SSE bridge that streams a task's transcript live. Tails the shadow
+ * JSONL that runTaskKickoffServerSide writes as the gateway stream
+ * yields events; that file mirrors OpenClaw's session.jsonl schema so
+ * the parser is the same one polling uses.
*
- * Used by the LiveTranscript component for in-flight tasks (especially
- * audit tasks that were auto-kicked-off server-side via the
- * blocked-by-task propagation hook — those have no /api/chat SSE
- * channel and would otherwise look "stuck" until JSONL flushes).
- *
- * Lifecycle:
- * - Resolves agent_full_id from the project's roster.
- * - Resolves the session key from threadId (the URL "label" segment
- * of OpenClaw's `agent:<agent>:<label>` keying).
- * - Opens a fresh GatewayClient (NOT the shared one — re-attach is
- * long-lived and would block other callers if it shared the
- * handshake), subscribes, listens, forwards.
- * - On client disconnect (browser close, navigation): tears down the
- * subscription and closes the gateway WS so we don't leak.
+ * Why not OpenClaw's WS-side `sessions.messages.subscribe`: that fires
+ * `session.message` events only when transcript-mirror appends a
+ * message, and the codex-app-server backend defers appending to
+ * session-end via mirrorTranscriptBestEffort. The result: zero events
+ * mid-turn. The shadow log fills that gap server-side.
*/
export async function GET(
request: Request,
{ params }: { params: Promise<{ agent: string; thread: string }> },
): Promise<Response> {
const { agent: agentSlug, thread: threadId } = await params;
+ log("GET", { agentSlug, threadId });
const url = new URL(request.url);
const projectSlug = url.searchParams.get("project");
const projectRow = projectSlug ? getProject(projectSlug) : await getActiveProject();
if (!projectRow || projectRow.archived_at) {
+ log("project not found", { projectSlug });
return NextResponse.json({ error: "Unknown project" }, { status: 404 });
}
const resolved = await resolveAgentBySlug(projectRow.slug, agentSlug);
if (!resolved) {
+ log("agent not found", { projectSlug: projectRow.slug, agentSlug });
return NextResponse.json({ error: "Unknown agent" }, { status: 404 });
}
const agentFullId = resolved.agent_id;
-
- // Resolve session key. If OpenClaw doesn't have a session row yet
- // (brand-new thread — first message hasn't been sent), fall back to
- // the pending key shape. Subscribing to a not-yet-existing session is
- // a no-op from the gateway's side; events will start flowing as soon
- // as the session is created.
- const sessionRow = findSessionBySessionId(agentFullId, threadId);
- const sessionKey =
- sessionRow?.sessionKey ?? buildPendingSessionKey(agentFullId, threadId);
+ const path = shadowTranscriptPath(agentFullId, threadId);
+ log("ready to tail", { agentFullId, threadId, path });
const encoder = new TextEncoder();
- let evtCounter = 0;
const stream = new ReadableStream<Uint8Array>({
async start(controller) {
- let client: GatewayClient | null = null;
- let removeListener: (() => void) | null = null;
let closed = false;
+ let byteOffset = 0;
+ let watcher: ReturnType<typeof watch> | null = null;
+ let pollTimer: ReturnType<typeof setInterval> | null = null;
+ let pumping = false;
+ let forwardedCount = 0;
function send(event: string, data: unknown): void {
if (closed) return;
@@ -82,24 +71,18 @@ export async function GET(
encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`),
);
} catch {
- // Stream already torn down; ignore.
+ // stream torn down
}
}
function teardown(): void {
if (closed) return;
closed = true;
- removeListener?.();
- // Best-effort unsubscribe; gateway times the subscription out
- // anyway when the WS closes.
- if (client?.isOpen()) {
- void client.request("sessions.messages.unsubscribe", {
- key: sessionKey,
- }).catch(() => {});
- }
+ log("teardown", { forwarded: forwardedCount, byteOffset });
try {
- client?.close();
+ watcher?.close();
} catch {}
+ if (pollTimer) clearInterval(pollTimer);
try {
controller.close();
} catch {}
@@ -107,57 +90,71 @@ export async function GET(
request.signal.addEventListener("abort", teardown);
- try {
- client = new GatewayClient();
- await client.open();
- removeListener = client.addEventListener((evt) => {
- if (evt.event !== "session.message") return;
- const payload = evt.payload as
- | {
- sessionKey?: string;
- message?: unknown;
- messageId?: string;
- messageSeq?: number;
- }
- | undefined;
- if (!payload) return;
- // Filter cross-session noise — subscriptions are scoped but the
- // gateway broadcasts on a shared listener pool.
- if (payload.sessionKey !== sessionKey) return;
-
- // The `message` payload mirrors the JSONL "message" field; wrap
- // it in a RawEntry so the shared parser produces the same shape
- // the polling path emits. Stable id from messageId/seq keeps
- // the client-side dedup honest (the same event can land via
- // SSE here AND via polling once JSONL flushes).
- const idHint =
- payload.messageId ??
- (typeof payload.messageSeq === "number"
- ? `msg-${payload.messageSeq}`
- : `sse-${evtCounter++}`);
- const raw: RawEntry = {
- type: "message",
- id: idHint,
- message: payload.message as RawEntry["message"],
- };
- const events = rawEntryToEvents(raw, idHint);
- if (events.length === 0) return;
- send("transcript", { events });
- });
+ async function pump(): Promise<void> {
+ if (closed) return;
+ if (pumping) return;
+ pumping = true;
+ try {
+ const { bytes, byteOffset: nextOffset } = await readShadowFromOffset(
+ agentFullId,
+ threadId,
+ byteOffset,
+ );
+ if (bytes.length === 0) return;
+ byteOffset = nextOffset;
+ const lines = bytes.split("\n").filter((l) => l.trim().length > 0);
+ let lineIdx = forwardedCount;
+ const events = [];
+ for (const line of lines) {
+ try {
+ const entry = JSON.parse(line) as Parameters<
+ typeof rawEntryToEvents
+ >[0];
+ const baseId = `${entry.id ?? "shadow"}-${lineIdx++}`;
+ events.push(...rawEntryToEvents(entry, baseId));
+ } catch {
+ // skip malformed
+ }
+ }
+ if (events.length > 0) {
+ forwardedCount += events.length;
+ log("forwarded", { count: events.length, total: forwardedCount });
+ send("transcript", { events });
+ }
+ } catch (err) {
+ log("pump error", err);
+ } finally {
+ pumping = false;
+ }
+ }
- // Subscribe + send a hello so the client knows the bridge is live.
- await client.request("sessions.messages.subscribe", { key: sessionKey });
- send("ready", { sessionKey });
+ try {
+ send("ready", { path });
+ // Initial pump in case the file already has content.
+ await pump();
+ // fs.watch fires inotify/FSEvents on append. The kernel notification
+ // can lag a few ms; we also poll every 500ms as a safety net so a
+ // missed inotify event (notoriously flaky on some filesystems)
+ // doesn't strand the user.
+ try {
+ watcher = watch(path, { persistent: true }, () => {
+ void pump();
+ });
+ } catch {
+ // File doesn't exist yet — runTaskKickoffServerSide will create
+ // it once the task is kicked off. Rely on poll fallback.
+ log("watch failed, polling only");
+ }
+ pollTimer = setInterval(() => {
+ void pump();
+ }, 500);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
+ log("error", { message });
send("error", { message });
teardown();
}
},
- cancel() {
- // Browser closed the EventSource — nothing else to do; the abort
- // signal handler already runs teardown.
- },
});
return new Response(stream, {
diff --git a/src/components/live-transcript.tsx b/src/components/live-transcript.tsx
index ad093e22..b867daa6 100644
--- a/src/components/live-transcript.tsx
+++ b/src/components/live-transcript.tsx
@@ -46,6 +46,32 @@ import { projectHref } from "@/lib/project-href";
const POLL_INTERVAL_MS = 2_000;
+/**
+ * Content signature for cross-writer dedup. The id-based set in
+ * seenEventIdsRef catches in-channel dups (e.g. polling racing itself);
+ * this catches dups WHEN the same logical event arrives via different
+ * channels with different ids — e.g. the shadow transcript stream gave
+ * us an assistant turn during the run, and OpenClaw's eventual JSONL
+ * flush gives us the same turn with a fresh UUID at session-end.
+ *
+ * Signature keys are intentionally narrow: text body for messages
+ * (case-insensitive trim collapses whitespace differences from the
+ * shadow vs. final renders), tool_call_id for tool rows (those are
+ * unique per invocation regardless of the writer).
+ */
+function eventSignature(e: TranscriptEvent): string {
+ switch (e.kind) {
+ case "user_message":
+ case "assistant_text":
+ return `${e.kind}|${e.body.trim()}`;
+ case "tool_call":
+ case "tool_result":
+ return `${e.kind}|${e.tool_call_id}`;
+ default:
+ return `${e.kind}|${e.id}`;
+ }
+}
+
type Props = {
projectSlug: string;
agentSlug: string;
@@ -194,6 +220,21 @@ export function LiveTranscript({
new Set(initialEvents.map((e) => e.id)),
);
+ /**
+ * Content signatures we've already rendered, for cross-writer dedup.
+ * The id-based set above can't catch the case where the SAME logical
+ * message arrives twice via different paths with different ids — the
+ * server-side-kickoff path writes a shadow transcript (ids like
+ * `shadow-<uuid>`), and OpenClaw later flushes its own copy of the
+ * session.jsonl at session-end with FRESH UUIDs. Without a content
+ * key, every assistant turn and tool call would render twice. Keyed by
+ * the semantic identity: text body for messages, tool_call_id for
+ * tool rows. Pure ref so React doesn't see it.
+ */
+ const seenSignaturesRef = useRef<Set<string>>(
+ new Set(initialEvents.map(eventSignature)),
+ );
+
// ── Auto-scroll: only when the user is already near the bottom. ─────
function onScroll() {
const el = scrollRef.current;
@@ -222,13 +263,19 @@ export function LiveTranscript({
byteOffset: number;
file_size: number;
};
- // Dedupe against `seenEventIdsRef` synchronously. The ref tracks
- // every id we've committed to state across the entire polling and
- // SSE lifetimes; rationale lives on the ref's declaration.
- const fresh = data.events.filter(
- (e) => !seenEventIdsRef.current.has(e.id),
- );
- for (const e of fresh) seenEventIdsRef.current.add(e.id);
+ // Dedupe against both id (in-channel races) and content signature
+ // (cross-channel dups, e.g. shadow stream vs. OpenClaw's eventual
+ // JSONL flush). Rationale on the refs' declarations.
+ const fresh = data.events.filter((e) => {
+ if (seenEventIdsRef.current.has(e.id)) return false;
+ const sig = eventSignature(e);
+ if (seenSignaturesRef.current.has(sig)) return false;
+ return true;
+ });
+ for (const e of fresh) {
+ seenEventIdsRef.current.add(e.id);
+ seenSignaturesRef.current.add(eventSignature(e));
+ }
if (fresh.length > 0) {
setEvents((prev) => [...prev, ...fresh]);
// Clear pending state: JSONL now has the canonical events so the
@@ -287,43 +334,74 @@ export function LiveTranscript({
// Skipped during an active /api/chat send — that path already streams
// its own deltas; layering re-attach on top would just duplicate work.
useEffect(() => {
- if (!composerDisabled) return;
- if (sendingChat) return;
- if (stopPolling) return;
- // jsdom (vitest) doesn't ship EventSource. Skip the bridge there so
- // component tests don't blow up; polling-path coverage stays intact.
- if (typeof EventSource === "undefined") return;
+ // Comprehensive client-side trace for the SSE re-attach. Prefix
+ // [live-bridge] mirrors the server-side logs so a browser-console +
+ // dev-server pair shows the full path. Toggle off in prod later.
+ const log = (...args: unknown[]) =>
+ console.log("[live-bridge]", ...args);
+ if (!composerDisabled) {
+ log("skip: composerDisabled=false (task not in flight)", {
+ threadId,
+ });
+ return;
+ }
+ if (sendingChat) {
+ log("skip: sendingChat=true (/api/chat path owns streaming)", {
+ threadId,
+ });
+ return;
+ }
+ if (stopPolling) {
+ log("skip: stopPolling=true", { threadId });
+ return;
+ }
+ if (typeof EventSource === "undefined") {
+ log("skip: no EventSource in env (jsdom test)");
+ return;
+ }
const url = `/api/agents/${agentSlug}/threads/${threadId}/live?project=${encodeURIComponent(projectSlug)}`;
+ log("opening", { url });
const es = new EventSource(url);
+ es.addEventListener("open", () => log("opened"));
+ es.addEventListener("ready", (e: MessageEvent) => log("ready", e.data));
es.addEventListener("transcript", (e: MessageEvent) => {
try {
const data = JSON.parse(e.data) as { events: TranscriptEvent[] };
- if (!Array.isArray(data.events)) return;
- const fresh = data.events.filter(
- (ev) => !seenEventIdsRef.current.has(ev.id),
- );
- for (const ev of fresh) seenEventIdsRef.current.add(ev.id);
+ if (!Array.isArray(data.events)) {
+ log("transcript payload not an array", data);
+ return;
+ }
+ const fresh = data.events.filter((ev) => {
+ if (seenEventIdsRef.current.has(ev.id)) return false;
+ const sig = eventSignature(ev);
+ if (seenSignaturesRef.current.has(sig)) return false;
+ return true;
+ });
+ log("transcript", {
+ incoming: data.events.length,
+ fresh: fresh.length,
+ });
+ for (const ev of fresh) {
+ seenEventIdsRef.current.add(ev.id);
+ seenSignaturesRef.current.add(eventSignature(ev));
+ }
if (fresh.length > 0) {
setEvents((prev) => [...prev, ...fresh]);
- // Live events arrived: clear any optimistic placeholders for
- // active sends + reset the "I'm waiting" indicator state.
setPendingUserMsg(null);
setPendingAssistant("");
setPendingTools([]);
setPendingError(null);
setPendingLifecycle(null);
}
- } catch {
- // Malformed payload; skip silently.
+ } catch (err) {
+ log("transcript parse error", err);
}
});
- es.addEventListener("error", () => {
- // EventSource auto-reconnects on transient drops; nothing else
- // needed here. If the server actually returned an error response
- // (4xx/5xx), reconnect attempts will keep failing and the source
- // closes itself — polling continues regardless.
+ es.addEventListener("error", (e) => {
+ log("error", { readyState: es.readyState, event: e });
});
return () => {
+ log("closing", { threadId });
es.close();
};
}, [
diff --git a/src/server/openclaw/shadow-transcript.ts b/src/server/openclaw/shadow-transcript.ts
new file mode 100644
index 00000000..e027c4e2
--- /dev/null
+++ b/src/server/openclaw/shadow-transcript.ts
@@ -0,0 +1,272 @@
+import { mkdir, appendFile, stat, readFile } from "node:fs/promises";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { randomUUID } from "node:crypto";
+
+import type { ChatStreamEvent } from "@/server/openclaw/gateway-client";
+
+/**
+ * Per-thread shadow transcript that runTaskKickoffServerSide writes as
+ * gateway-stream events arrive. Exists because OpenClaw's codex-app-server
+ * backend buffers the agent's whole turn in memory and writes the
+ * session.jsonl in one shot at session-end via mirrorTranscriptBestEffort
+ * — which means `sessions.messages.subscribe` over the WS gateway never
+ * fires `session.message` events DURING a turn, even though the agent is
+ * producing tokens. Our shadow log fills that gap: we append OpenClaw-
+ * shaped lines as soon as we see each delta / tool call / tool result,
+ * the SSE bridge tails this file with fs.watch, and the browser renders
+ * tokens live just like /api/chat already does.
+ *
+ * The format is intentionally a subset of OpenClaw's session.jsonl
+ * schema so readTranscriptTail's existing parser (rawEntryToEvents)
+ * works on both files. The same TranscriptEvent.id space means the
+ * client's seenEventIdsRef dedups when OpenClaw eventually flushes its
+ * own copy of the transcript at session-end.
+ *
+ * Location: `~/.notfair-cmo/agents/<agentFullId>/shadow/<threadId>.jsonl`.
+ * Living under .notfair-cmo (not .openclaw) so OpenClaw's writers can't
+ * race ours; readTranscriptTail explicitly merges both files.
+ */
+
+function dataDir(): string {
+ return process.env.NOTFAIR_CMO_DATA_DIR ?? join(homedir(), ".notfair-cmo");
+}
+
+export function shadowTranscriptPath(
+ agentFullId: string,
+ threadId: string,
+): string {
+ return join(dataDir(), "agents", agentFullId, "shadow", `${threadId}.jsonl`);
+}
+
+export function shadowTranscriptExists(
+ agentFullId: string,
+ threadId: string,
+): boolean {
+ return existsSync(shadowTranscriptPath(agentFullId, threadId));
+}
+
+export async function ensureShadowDir(
+ agentFullId: string,
+ threadId: string,
+): Promise<void> {
+ await mkdir(dirname(shadowTranscriptPath(agentFullId, threadId)), {
+ recursive: true,
+ });
+}
+
+/**
+ * State machine the gateway-stream consumer keeps per turn. Deltas
+ * accumulate into a single assistant text event committed when the
+ * stream emits `final` (or the turn ends). Tool starts/results write
+ * dedicated entries with the gateway's toolCallId so the client can
+ * pair them.
+ */
+export type ShadowWriter = {
+ /** Append `assistantBuffer` as a single assistant message + clear. */
+ flushAssistant(): Promise<void>;
+ /** Push another delta of token text onto the assistant buffer. */
+ appendDelta(text: string): void;
+ /**
+ * Replace the assistant buffer with `text`. Used on the gateway's
+ * `final` event — for providers that emit BOTH per-token deltas and
+ * a cumulative final, appending would double-count. Replace makes the
+ * final authoritative.
+ */
+ replaceAssistant(text: string): void;
+ /** Record a tool call start. */
+ toolStart(toolCallId: string, name: string, label?: string): Promise<void>;
+ /** Record a tool result (success or error). */
+ toolResult(
+ toolCallId: string,
+ name: string,
+ ok: boolean,
+ summary?: string,
+ ): Promise<void>;
+ /** Close the writer — flushes any pending assistant text. */
+ close(): Promise<void>;
+};
+
+export async function openShadowWriter(
+ agentFullId: string,
+ threadId: string,
+): Promise<ShadowWriter> {
+ await ensureShadowDir(agentFullId, threadId);
+ const path = shadowTranscriptPath(agentFullId, threadId);
+ let assistantBuffer = "";
+
+ async function writeEntry(entry: object): Promise<void> {
+ await appendFile(path, `${JSON.stringify(entry)}\n`, "utf8");
+ }
+
+ async function flushAssistant(): Promise<void> {
+ const body = assistantBuffer.trim();
+ assistantBuffer = "";
+ if (!body) return;
+ await writeEntry({
+ type: "message",
+ id: `shadow-${randomUUID()}`,
+ timestamp: new Date().toISOString(),
+ message: {
+ role: "assistant",
+ content: [{ type: "text", text: body }],
+ },
+ });
+ }
+
+ return {
+ appendDelta(text: string) {
+ assistantBuffer += text;
+ },
+ replaceAssistant(text: string) {
+ assistantBuffer = text;
+ },
+ flushAssistant,
+ async toolStart(toolCallId, name, label) {
+ // Flush any preceding assistant text so the tool call shows in the
+ // right order in the transcript.
+ await flushAssistant();
+ await writeEntry({
+ type: "message",
+ id: `shadow-${randomUUID()}`,
+ timestamp: new Date().toISOString(),
+ message: {
+ role: "assistant",
+ content: [
+ {
+ type: "toolCall",
+ id: toolCallId,
+ name,
+ ...(label ? { arguments: { __label: label } } : {}),
+ },
+ ],
+ },
+ });
+ },
+ async toolResult(toolCallId, name, ok, summary) {
+ await writeEntry({
+ type: "message",
+ id: `shadow-${randomUUID()}`,
+ timestamp: new Date().toISOString(),
+ message: {
+ role: "toolResult",
+ toolCallId,
+ toolName: name,
+ isError: !ok,
+ content: summary
+ ? [{ type: "text", text: summary }]
+ : undefined,
+ },
+ });
+ },
+ async close() {
+ await flushAssistant();
+ },
+ };
+}
+
+/**
+ * Hook into runTaskKickoffServerSide's stream: forward each event to
+ * the shadow writer with reasonable bucketing. Returns a function that
+ * closes the writer (flush any buffered assistant text).
+ */
+export async function shadowStreamEvent(
+ writer: ShadowWriter,
+ evt: ChatStreamEvent,
+): Promise<void> {
+ switch (evt.kind) {
+ case "delta":
+ writer.appendDelta(evt.text);
+ return;
+ case "final":
+ // `final.text` is the cumulative assistant turn — for providers
+ // that ALSO emit per-token deltas, naive appending would
+ // double-count the same text. Replace the buffer with `final`
+ // and flush exactly once. Providers that only emit final
+ // (deltaless backends) land in the same place.
+ writer.replaceAssistant(evt.text);
+ await writer.flushAssistant();
+ return;
+ case "tool":
+ if (evt.phase === "start") {
+ await writer.toolStart(evt.toolCallId, evt.name, evt.label);
+ } else if (evt.phase === "result") {
+ // We don't know ok/summary from the result event shape; treat
+ // as ok with no summary. Errors will surface via the gateway's
+ // own `error` events.
+ await writer.toolResult(evt.toolCallId, evt.name, true, evt.label);
+ }
+ return;
+ case "lifecycle":
+ case "error":
+ return;
+ }
+}
+
+/**
+ * Read the shadow JSONL from `byteOffset` to EOF. Mirrors the API of
+ * readTranscriptTail so the polling path can merge shadow + OpenClaw
+ * transcripts.
+ */
+export type ShadowTailResult = {
+ bytes: string;
+ byteOffset: number;
+ fileSize: number;
+};
+
+export function readShadowTail(
+ agentFullId: string,
+ threadId: string,
+ byteOffset: number,
+): ShadowTailResult {
+ const path = shadowTranscriptPath(agentFullId, threadId);
+ if (!existsSync(path)) return { bytes: "", byteOffset, fileSize: 0 };
+ let size: number;
+ try {
+ size = statSync(path).size;
+ } catch {
+ return { bytes: "", byteOffset, fileSize: 0 };
+ }
+ if (byteOffset > size) byteOffset = 0;
+ if (byteOffset === size) return { bytes: "", byteOffset, fileSize: size };
+ let raw: string;
+ try {
+ const buf = readFileSync(path);
+ raw = buf.slice(byteOffset).toString("utf8");
+ } catch {
+ return { bytes: "", byteOffset, fileSize: size };
+ }
+ const lastNl = raw.lastIndexOf("\n");
+ const consumable = lastNl < 0 ? "" : raw.slice(0, lastNl + 1);
+ const newByteOffset = byteOffset + Buffer.byteLength(consumable, "utf8");
+ return { bytes: consumable, byteOffset: newByteOffset, fileSize: size };
+}
+
+/** Async read for SSE tailer that wants non-blocking IO. */
+export async function readShadowFromOffset(
+ agentFullId: string,
+ threadId: string,
+ byteOffset: number,
+): Promise<ShadowTailResult> {
+ const path = shadowTranscriptPath(agentFullId, threadId);
+ let size: number;
+ try {
+ size = (await stat(path)).size;
+ } catch {
+ return { bytes: "", byteOffset, fileSize: 0 };
+ }
+ if (byteOffset > size) byteOffset = 0;
+ if (byteOffset === size) return { bytes: "", byteOffset, fileSize: size };
+ let raw: string;
+ try {
+ const buf = await readFile(path);
+ raw = buf.slice(byteOffset).toString("utf8");
+ } catch {
+ return { bytes: "", byteOffset, fileSize: size };
+ }
+ const lastNl = raw.lastIndexOf("\n");
+ const consumable = lastNl < 0 ? "" : raw.slice(0, lastNl + 1);
+ const newByteOffset = byteOffset + Buffer.byteLength(consumable, "utf8");
+ return { bytes: consumable, byteOffset: newByteOffset, fileSize: size };
+}
diff --git a/src/server/orchestration/run-task.test.ts b/src/server/orchestration/run-task.test.ts
index 7d3e5330..e5bc172c 100644
--- a/src/server/orchestration/run-task.test.ts
+++ b/src/server/orchestration/run-task.test.ts
@@ -23,6 +23,20 @@ vi.mock("@/server/openclaw/sessions", () => ({
`agent:${agent}:${thread}`,
}));
+// Shadow transcript writes to ~/.notfair-cmo by default — stub to keep
+// the unit test isolated from disk. Functional behavior is covered by
+// the e2e browser test.
+vi.mock("@/server/openclaw/shadow-transcript", () => ({
+ openShadowWriter: async () => ({
+ appendDelta: () => {},
+ flushAssistant: async () => {},
+ toolStart: async () => {},
+ toolResult: async () => {},
+ close: async () => {},
+ }),
+ shadowStreamEvent: async () => {},
+}));
+
// generateTaskThreadId lives on task-kickoff now. Mock just that one symbol
// while letting the real buildTaskKickoffMessage run — we want to catch
// contract drift between run-task and the kickoff format.
diff --git a/src/server/orchestration/run-task.ts b/src/server/orchestration/run-task.ts
index 3fe472a4..5685418d 100644
--- a/src/server/orchestration/run-task.ts
+++ b/src/server/orchestration/run-task.ts
@@ -2,6 +2,10 @@ import {
buildPendingSessionKey,
} from "@/server/openclaw/sessions";
import { streamChatViaGateway } from "@/server/openclaw/gateway-client";
+import {
+ openShadowWriter,
+ shadowStreamEvent,
+} from "@/server/openclaw/shadow-transcript";
import { claimProposedTask, setTaskThreadIfMissing, updateTask } from "@/server/db/tasks";
import type { Task } from "@/types";
@@ -57,6 +61,15 @@ export async function runTaskKickoffServerSide(task: Task): Promise<void> {
const sessionKey = buildPendingSessionKey(finalTask.agent_id, finalTask.thread_id);
const kickoffMessage = buildTaskKickoffMessage(finalTask);
+ // Tee gateway events to a shadow JSONL the browser can tail. OpenClaw's
+ // codex-app-server backend buffers session.jsonl until session-end and
+ // doesn't emit per-message broadcasts during the turn, so without the
+ // shadow there's no way for the UI to render tokens live for tasks
+ // kicked off server-side. The shadow uses the same OpenClaw-style
+ // schema as session.jsonl so transcript-tail can read both with the
+ // same parser.
+ const shadow = await openShadowWriter(finalTask.agent_id, finalTask.thread_id);
+
// We drain the stream so the agent's turn fully runs, but we no longer
// post-process the buffer for pseudo-XML blocks — any side effects
// (task_status, comments, approvals) happen via the agent calling
@@ -67,11 +80,20 @@ export async function runTaskKickoffServerSide(task: Task): Promise<void> {
sessionId: finalTask.thread_id,
message: kickoffMessage,
})) {
+ // Best-effort forward to the shadow; a shadow-write failure must
+ // not abort the actual gateway stream.
+ try {
+ await shadowStreamEvent(shadow, evt);
+ } catch (err) {
+ console.error("[run-task] shadow write failed:", err);
+ }
if (evt.kind === "error") {
throw new Error(evt.message);
}
}
+ await shadow.close();
} catch (err) {
+ await shadow.close().catch(() => undefined);
const message = err instanceof Error ? err.message : String(err);
console.error(`[run-task] kickoff failed for ${finalTask.id}:`, err);
updateTask(finalTask.id, {