Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(daemon): keep slow inbox pulls replayable (#717)
Gustavo Yed1d1d0bddf2d
diff --git a/src/daemon/src/cli/index.test.ts b/src/daemon/src/cli/index.test.tsindex 9db1f7546..0a7bb4062 100644--- a/src/daemon/src/cli/index.test.ts+++ b/src/daemon/src/cli/index.test.ts@@ -528,6 +528,92 @@ describe("inbox pull", () => {expect(env.success.pulledAt).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{2}:\d{2}$/);});+ it("leaves a pull near the outer 10s wait unread so a fast retry can replay and acknowledge it", async () => {+ const message = {+ seq: "#7",+ channel: "/s#0042/general",+ sender: "@x",+ content: { text: "replay me" },+ time: "",+ };+ let readSeq = 0;+ const ackSpy = vi.fn(async (req: { cursors: Array<{ channel: string; seq: number }> }) => {+ readSeq = Math.max(readSeq, ...req.cursors.map((cursor) => cursor.seq));+ return { ok: true, applied: req.cursors, failed: [] };+ });+ const nowSpy = vi.spyOn(performance, "now")+ .mockReturnValueOnce(100)+ .mockReturnValueOnce(9_900)+ .mockReturnValueOnce(20_000)+ .mockReturnValueOnce(20_001);+ setApiForTesting(+ stubApi({+ inboxPull: async () => ({+ messages: readSeq >= 7 ? [] : [message],+ hasMore: false,+ markedCount: 0,+ }),+ ack: ackSpy,+ }),+ );++ try {+ await main(["inbox", "pull"]);+ expect(ackSpy).not.toHaveBeenCalled();+ const slow = parseEnvelope(cap.lines()) as {+ success: {+ acked: number;+ messages: unknown[];+ ackDeferred?: { reason: string; thresholdMs: number; elapsedMs: number };+ };+ };+ expect(slow.success.messages).toEqual([message]);+ expect(slow.success.acked).toBe(0);+ expect(slow.success.ackDeferred).toEqual({+ reason: "slow_pull",+ thresholdMs: 9_000,+ elapsedMs: 9_800,+ });++ cap.lines().length = 0;+ await main(["inbox", "pull"]);+ } finally {+ nowSpy.mockRestore();+ }++ const retry = parseEnvelope(cap.lines()) as {+ success: { acked: number; messages: unknown[]; ackDeferred?: unknown };+ };+ expect(ackSpy).toHaveBeenCalledOnce();+ expect(retry.success.messages).toEqual([message]);+ expect(retry.success.acked).toBe(1);+ expect(retry.success.ackDeferred).toBeUndefined();+ });++ it("does not report deferred acknowledgement for an empty slow pull", async () => {+ const ackSpy = vi.fn();+ const nowSpy = vi.spyOn(performance, "now")+ .mockReturnValueOnce(0)+ .mockReturnValueOnce(20_000);+ setApiForTesting(stubApi({+ inboxPull: async () => ({ messages: [], hasMore: false, markedCount: 0 }),+ ack: ackSpy,+ }));++ try {+ await main(["inbox", "pull"]);+ } finally {+ nowSpy.mockRestore();+ }++ const env = parseEnvelope(cap.lines()) as {+ success: { acked: number; ackDeferred?: unknown };+ };+ expect(ackSpy).not.toHaveBeenCalled();+ expect(env.success.acked).toBe(0);+ expect(env.success.ackDeferred).toBeUndefined();+ });+it("removes --no-ack from inbox pull help", async () => {await main(["inbox", "pull", "-h"]);expect(cap.lines().join("")).not.toContain("--no-ack");diff --git a/src/daemon/src/cli/index.ts b/src/daemon/src/cli/index.tsindex 9d50e1c68..975375e24 100755--- a/src/daemon/src/cli/index.ts+++ b/src/daemon/src/cli/index.ts@@ -209,6 +209,10 @@ function stripMalformedAlookHeredocTail(input: string): string {const CLIENT_MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024;const CLIENT_MAX_MESSAGE_BODY_BYTES = 1024;+// The agent tool stops waiting after 10 seconds. Do not start an acknowledgement+// once the pull has consumed the first nine, leaving the normal ack/output path+// one second of headroom at that outer boundary.+const INBOX_PULL_OUTPUT_DEADLINE_MS = 9_000;/*** Guess a content-type from a filename extension. Kept trivial — the server@@ -597,12 +601,15 @@ async function cmdInboxPull(opts: Record<string, unknown>): Promise<unknown> {const api = getApi();const agent = agentId(opts);const max = opts.max ? Number(opts.max) : undefined;+ const pullStartedAt = performance.now();const { messages, hasMore, markedCount } = await api.inboxPull({ agentId: agent, max });+ const pullElapsedMs = performance.now() - pullStartedAt;const pulledAt = nowLocalISO();let acked = 0;let ackError: string | undefined;let failed: AckFailure[] | undefined;+ let ackDeferred: { reason: "slow_pull"; thresholdMs: number; elapsedMs: number } | undefined;if (messages.length > 0) {const latest = new Map<string, Cursor>();for (const m of messages) {@@ -610,18 +617,22 @@ async function cmdInboxPull(opts: Record<string, unknown>): Promise<unknown> {const cur = latest.get(m.channel);if (!cur || seqN > cur.seq) latest.set(m.channel, { channel: m.channel, seq: seqN });}- try {- const result = await api.ack({ agentId: agent, cursors: [...latest.values()] });- acked = result.applied.length;- failed = result.failed;- } catch (err) {- // Do NOT rethrow: the pull already succeeded, and if ack fails on a- // single scope (e.g. a stale visibility mismatch) the whole envelope- // would otherwise collapse to a bare error, wiping the messages the- // agent needs. Surface the ack failure separately so the agent (or a- // human debugging) sees BOTH the delivered messages AND that the- // waterline didn't move.- ackError = err instanceof Error ? err.message : String(err);+ if (pullElapsedMs >= INBOX_PULL_OUTPUT_DEADLINE_MS) {+ ackDeferred = {+ reason: "slow_pull",+ thresholdMs: INBOX_PULL_OUTPUT_DEADLINE_MS,+ elapsedMs: Math.round(pullElapsedMs),+ };+ } else {+ try {+ const result = await api.ack({ agentId: agent, cursors: [...latest.values()] });+ acked = result.applied.length;+ failed = result.failed;+ } catch (err) {+ // Keep the successfully pulled messages visible even when the+ // automatic acknowledgement fails.+ ackError = err instanceof Error ? err.message : String(err);+ }}}@@ -632,6 +643,7 @@ async function cmdInboxPull(opts: Record<string, unknown>): Promise<unknown> {pulledAt,...(failed ? { failed } : {}),...(ackError ? { ackError } : {}),+ ...(ackDeferred ? { ackDeferred } : {}),...(markedCount > 0? {markedReminder: `You have ${markedCount} marked ${markedCount === 1 ? "message" : "messages"}. Resolve ${markedCount === 1 ? "it" : "them"} before going dark unless blocked.`,diff --git a/src/daemon/src/credentials/credentialProxy.test.ts b/src/daemon/src/credentials/credentialProxy.test.tsindex eb371e0ed..cfe680083 100644--- a/src/daemon/src/credentials/credentialProxy.test.ts+++ b/src/daemon/src/credentials/credentialProxy.test.ts@@ -738,6 +738,175 @@ describe("startCredentialProxy (zero-trust end to end)", () => {expect(onInboxPullObservationError).not.toHaveBeenCalled();});+ it("observes only server-applied cursors from a successful inbox ack", async () => {+ const applied = [+ { channel: "/s#0001/c", seq: 3 },+ { channel: "/s#0001/private", seq: 8 },+ ];+ const body = JSON.stringify({+ ok: false,+ applied,+ failed: [{ channel: "/s#0001/blocked", seq: 5, code: "forbidden", error: "forbidden" }],+ });+ const upstream = await startUpstream({ body });+ upstreamClose = upstream.close;+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });+ const onInboxAckStart = vi.fn(() => ({ generation: 4 }));+ const onInboxAckResponse = vi.fn();+ const onInboxAckObservationError = vi.fn();+ proxy = await startCredentialProxy(broker, {+ onInboxAckStart,+ onInboxAckResponse,+ onInboxAckObservationError,+ });+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);++ const response = await post(+ proxy.url,+ reg.voucher,+ "/api/community/users/me/inbox/ack",+ { "accept-encoding": "gzip, deflate" },+ );++ expect(response).toEqual({ status: 200, body });+ expect(upstream.seen[0]?.acceptEncoding).toBe("identity");+ expect(onInboxAckStart).toHaveBeenCalledWith("agent-1");+ expect(onInboxAckResponse).toHaveBeenCalledOnce();+ expect(onInboxAckResponse).toHaveBeenCalledWith("agent-1", applied, { generation: 4 });+ expect(onInboxAckObservationError).not.toHaveBeenCalled();+ });++ it("does not observe a failed inbox ack response", async () => {+ const body = JSON.stringify({ ok: true, applied: [{ channel: "/s#0001/c", seq: 3 }], failed: [] });+ const upstream = await startUpstream({ status: 500, body });+ upstreamClose = upstream.close;+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });+ const onInboxAckResponse = vi.fn();+ const onInboxAckObservationError = vi.fn();+ proxy = await startCredentialProxy(broker, { onInboxAckResponse, onInboxAckObservationError });+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);++ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");++ expect(response).toEqual({ status: 500, body });+ expect(onInboxAckResponse).not.toHaveBeenCalled();+ expect(onInboxAckObservationError).not.toHaveBeenCalled();+ });++ it("forwards a malformed successful ack body without advancing model-seen", async () => {+ const body = JSON.stringify({ ok: true, applied: [{ channel: "/s#0001/c", seq: "3" }] });+ const upstream = await startUpstream({ body });+ upstreamClose = upstream.close;+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });+ const onInboxAckResponse = vi.fn();+ const onInboxAckObservationError = vi.fn();+ proxy = await startCredentialProxy(broker, { onInboxAckResponse, onInboxAckObservationError });+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);++ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");++ expect(response).toEqual({ status: 200, body });+ expect(onInboxAckResponse).not.toHaveBeenCalled();+ expect(onInboxAckObservationError).toHaveBeenCalledWith({+ agentId: "agent-1",+ reason: "invalid_ack_shape",+ contentEncoding: "identity",+ });+ });++ it("forwards inbox ack when the observation start hook throws", async () => {+ const body = JSON.stringify({ ok: true, applied: [], failed: [] });+ const upstream = await startUpstream({ body });+ upstreamClose = upstream.close;+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });+ const onInboxAckObservationError = vi.fn();+ proxy = await startCredentialProxy(broker, {+ onInboxAckStart: () => {+ throw new Error("start observer failed");+ },+ onInboxAckResponse: vi.fn(),+ onInboxAckObservationError,+ });+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);++ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");++ expect(response).toEqual({ status: 200, body });+ expect(onInboxAckObservationError).toHaveBeenCalledWith({+ agentId: "agent-1",+ reason: "observer_failed",+ contentEncoding: "identity",+ });+ });++ it("forwards an encoded inbox ack without attempting to parse it", async () => {+ const json = JSON.stringify({ ok: true, applied: [{ channel: "/s#0001/c", seq: 3 }], failed: [] });+ const upstream = await startUpstream({+ headers: { "content-encoding": "gzip" },+ body: gzipSync(json),+ });+ upstreamClose = upstream.close;+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });+ const onInboxAckResponse = vi.fn();+ const onInboxAckObservationError = vi.fn();+ proxy = await startCredentialProxy(broker, { onInboxAckResponse, onInboxAckObservationError });+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);++ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");++ expect(response).toEqual({ status: 200, body: json });+ expect(onInboxAckResponse).not.toHaveBeenCalled();+ expect(onInboxAckObservationError).toHaveBeenCalledWith({+ agentId: "agent-1",+ reason: "unexpected_content_encoding",+ contentEncoding: "gzip",+ });+ });++ it("keeps forwarding invalid ack JSON when the error observer also throws", async () => {+ const body = "not-json";+ const upstream = await startUpstream({ body });+ upstreamClose = upstream.close;+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });+ const onInboxAckResponse = vi.fn();+ proxy = await startCredentialProxy(broker, {+ onInboxAckResponse,+ onInboxAckObservationError: () => {+ throw new Error("error observer failed");+ },+ });+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);++ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");++ expect(response).toEqual({ status: 200, body });+ expect(onInboxAckResponse).not.toHaveBeenCalled();+ });++ it("keeps forwarding a valid ack when the response observer throws", async () => {+ const body = JSON.stringify({ ok: true, applied: [{ channel: "/s#0001/c", seq: 3 }], failed: [] });+ const upstream = await startUpstream({ body });+ upstreamClose = upstream.close;+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });+ const onInboxAckObservationError = vi.fn();+ proxy = await startCredentialProxy(broker, {+ onInboxAckResponse: () => {+ throw new Error("ack observer failed");+ },+ onInboxAckObservationError,+ });+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);++ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");++ expect(response).toEqual({ status: 200, body });+ expect(onInboxAckObservationError).toHaveBeenCalledWith({+ agentId: "agent-1",+ reason: "observer_failed",+ contentEncoding: "identity",+ });+ });+it("preserves compression negotiation for non-pull and wrong-method lookalike routes", async () => {const upstream = await startUpstream({ body: JSON.stringify({ messages: [{ seq: "#1" }] }) });upstreamClose = upstream.close;diff --git a/src/daemon/src/credentials/credentialProxy.ts b/src/daemon/src/credentials/credentialProxy.tsindex f34a564cc..ab866dc59 100644--- a/src/daemon/src/credentials/credentialProxy.ts+++ b/src/daemon/src/credentials/credentialProxy.ts@@ -44,7 +44,7 @@ import * as https from "https";import * as os from "os";import * as path from "path";import { URL } from "url";-import { formatRef, parseRef, type Message } from "../server/contract.js";+import { formatRef, parseRef, type Cursor, type Message } from "../server/contract.js";import { parseNameAndTag } from "@alook/shared/lib/discriminator";export const LOCAL_MESSAGE_REMINDER_PATH = "/__alook/local/message-reminder";@@ -318,6 +318,9 @@ export interface CredentialProxyOptions {onInboxPullStart?: (agentId: string) => unknown;onInboxPullResponse?: (agentId: string, messages: Message[], observationToken?: unknown) => void;onInboxPullObservationError?: (failure: InboxPullObservationFailure) => void;+ onInboxAckStart?: (agentId: string) => unknown;+ onInboxAckResponse?: (agentId: string, applied: Cursor[], observationToken?: unknown) => void;+ onInboxAckObservationError?: (failure: InboxAckObservationFailure) => void;/*** Called once per successfully-authorized upstream proxy request, BEFORE the* upstream is dispatched. Fires only on `verdict.ok === true` — never on@@ -364,6 +367,18 @@ export interface InboxPullObservationFailure {contentEncoding: InboxPullContentEncoding;}+export type InboxAckObservationFailureReason =+ | "unexpected_content_encoding"+ | "invalid_json"+ | "invalid_ack_shape"+ | "observer_failed";++export interface InboxAckObservationFailure {+ agentId: string;+ reason: InboxAckObservationFailureReason;+ contentEncoding: InboxPullContentEncoding;+}+export interface RunningProxy {url: string;port: number;@@ -510,6 +525,7 @@ export async function startCredentialProxy(const upstreamClient = upstream.protocol === "https:" ? https : http;const onPull = options.onInboxPullResponse;+ const onAck = options.onInboxAckResponse;const onProxyRequest = options.onProxyRequest;const reportInboxPullObservationError = (failure: InboxPullObservationFailure): void => {try {@@ -518,6 +534,13 @@ export async function startCredentialProxy(// Observational callback — never disrupt the proxy response.}};+ const reportInboxAckObservationError = (failure: InboxAckObservationFailure): void => {+ try {+ options.onInboxAckObservationError?.(failure);+ } catch {+ return;+ }+ };const server = http.createServer((req, res) => {const pathname = new URL(req.url ?? "/", "http://placeholder").pathname;@@ -581,8 +604,11 @@ export async function startCredentialProxy(// `/api/inboxPull` verb is deleted (flat-delete step). inboxSnapshot is// deliberately EXCLUDED (peek, no `{ messages }` to record).const isInboxPull = req.method === "POST" && pathname === "/api/community/users/me/inbox/pull";+ const isInboxAck = req.method === "POST" && pathname === "/api/community/users/me/inbox/ack";const shouldObserveInboxPull = Boolean(isInboxPull && onPull);+ const shouldObserveInboxAck = Boolean(isInboxAck && onAck);let inboxPullObservationToken: unknown;+ let inboxAckObservationToken: unknown;if (shouldObserveInboxPull) {try {inboxPullObservationToken = options.onInboxPullStart?.(reg.agentId);@@ -594,6 +620,17 @@ export async function startCredentialProxy(});}}+ if (shouldObserveInboxAck) {+ try {+ inboxAckObservationToken = options.onInboxAckStart?.(reg.agentId);+ } catch {+ reportInboxAckObservationError({+ agentId: reg.agentId,+ reason: "observer_failed",+ contentEncoding: "identity",+ });+ }+ }if (onProxyRequest) {try {@@ -616,7 +653,7 @@ export async function startCredentialProxy(outHeaders[broker.headerNames.agentId.toLowerCase()] = reg.agentId;outHeaders[broker.headerNames.client.toLowerCase()] = broker.clientLabel;outHeaders[broker.headerNames.capabilities.toLowerCase()] = [...reg.capabilities].join(",");- if (isInboxPull) outHeaders["accept-encoding"] = "identity";+ if (isInboxPull || isInboxAck) outHeaders["accept-encoding"] = "identity";// `responded` guards only against writing a second response to the// DOWNSTREAM client (writeHead/end can't fire twice) — it does NOT gate@@ -654,53 +691,97 @@ export async function startCredentialProxy(res_.on("error", destroyResIfIncomplete);res_.on("close", destroyResIfIncomplete);if (- shouldObserveInboxPull+ (shouldObserveInboxPull || shouldObserveInboxAck)&& res_.statusCode !== undefined&& res_.statusCode >= 200&& res_.statusCode < 300) {- // Buffer the inboxPull response to surface pulled messages to the daemon.+ // Buffer observed inbox responses so the daemon can track pull and ack boundaries.const chunks: Buffer[] = [];res_.on("data", (chunk: Buffer) => chunks.push(chunk));res_.on("end", () => {const body = Buffer.concat(chunks);const contentEncoding = normalizeContentEncoding(res_.headers["content-encoding"]);if (contentEncoding !== "identity") {- reportInboxPullObservationError({- agentId: reg.agentId,- reason: "unexpected_content_encoding",- contentEncoding,- });+ if (shouldObserveInboxPull) {+ reportInboxPullObservationError({+ agentId: reg.agentId,+ reason: "unexpected_content_encoding",+ contentEncoding,+ });+ } else {+ reportInboxAckObservationError({+ agentId: reg.agentId,+ reason: "unexpected_content_encoding",+ contentEncoding,+ });+ }} else {let parsed: unknown;try {parsed = JSON.parse(body.toString("utf8"));} catch {- reportInboxPullObservationError({- agentId: reg.agentId,- reason: "invalid_json",- contentEncoding,- });+ if (shouldObserveInboxPull) {+ reportInboxPullObservationError({ agentId: reg.agentId, reason: "invalid_json", contentEncoding });+ } else {+ reportInboxAckObservationError({ agentId: reg.agentId, reason: "invalid_json", contentEncoding });+ }}if (parsed !== undefined) {- if (+ if (shouldObserveInboxPull) {+ if (+ !parsed+ || typeof parsed !== "object"+ || Array.isArray(parsed)+ || !Array.isArray((parsed as { messages?: unknown }).messages)+ ) {+ reportInboxPullObservationError({+ agentId: reg.agentId,+ reason: "invalid_inbox_shape",+ contentEncoding,+ });+ } else {+ const messages = (parsed as { messages: Message[] }).messages;+ if (messages.length > 0) {+ try {+ onPull!(reg.agentId, messages, inboxPullObservationToken);+ } catch {+ reportInboxPullObservationError({+ agentId: reg.agentId,+ reason: "observer_failed",+ contentEncoding,+ });+ }+ }+ }+ } else if (!parsed|| typeof parsed !== "object"|| Array.isArray(parsed)- || !Array.isArray((parsed as { messages?: unknown }).messages)+ || !Array.isArray((parsed as { applied?: unknown }).applied)+ || !(parsed as { applied: unknown[] }).applied.every((cursor) => (+ Boolean(cursor)+ && typeof cursor === "object"+ && !Array.isArray(cursor)+ && typeof (cursor as { channel?: unknown }).channel === "string"+ && (cursor as { channel: string }).channel.length > 0+ && typeof (cursor as { seq?: unknown }).seq === "number"+ && Number.isSafeInteger((cursor as { seq: number }).seq)+ && (cursor as { seq: number }).seq > 0+ ))) {- reportInboxPullObservationError({+ reportInboxAckObservationError({agentId: reg.agentId,- reason: "invalid_inbox_shape",+ reason: "invalid_ack_shape",contentEncoding,});} else {- const messages = (parsed as { messages: Message[] }).messages;- if (messages.length > 0) {+ const applied = (parsed as { applied: Cursor[] }).applied;+ if (applied.length > 0) {try {- onPull!(reg.agentId, messages, inboxPullObservationToken);+ onAck!(reg.agentId, applied, inboxAckObservationToken);} catch {- reportInboxPullObservationError({+ reportInboxAckObservationError({agentId: reg.agentId,reason: "observer_failed",contentEncoding,diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.tsindex 2c7ceecc0..fa263dd56 100644--- a/src/daemon/src/daemon/createDaemon.test.ts+++ b/src/daemon/src/daemon/createDaemon.test.ts@@ -58,6 +58,11 @@ const credentialProxyHarness = vi.hoisted(() => ({((agentId: string, messages: unknown[], observationToken?: unknown) => void) | undefined),onInboxPullObservationError: undefined as ((failure: Record<string, unknown>) => void) | undefined,+ onInboxAckStart: undefined as ((agentId: string) => unknown) | undefined,+ onInboxAckResponse: undefined as (+ ((agentId: string, applied: Array<{ channel: string; seq: number }>, observationToken?: unknown) => void) | undefined+ ),+ onInboxAckObservationError: undefined as ((failure: Record<string, unknown>) => void) | undefined,onProxyRequest: undefined as ((agentId: string,method: string,@@ -102,6 +107,13 @@ vi.mock("../credentials/index.js", async (importOriginal) => {credentialProxyHarness.onInboxPullObservationError = args[1]?.onInboxPullObservationError as| ((failure: Record<string, unknown>) => void)| undefined;+ credentialProxyHarness.onInboxAckStart = args[1]?.onInboxAckStart;+ credentialProxyHarness.onInboxAckResponse = args[1]?.onInboxAckResponse as+ | ((agentId: string, applied: Array<{ channel: string; seq: number }>, observationToken?: unknown) => void)+ | undefined;+ credentialProxyHarness.onInboxAckObservationError = args[1]?.onInboxAckObservationError as+ | ((failure: Record<string, unknown>) => void)+ | undefined;credentialProxyHarness.onProxyRequest = args[1]?.onProxyRequest;return actual.startCredentialProxy(...args);},@@ -116,6 +128,9 @@ afterEach(() => {credentialProxyHarness.onInboxPullStart = undefined;credentialProxyHarness.onInboxPullResponse = undefined;credentialProxyHarness.onInboxPullObservationError = undefined;+ credentialProxyHarness.onInboxAckStart = undefined;+ credentialProxyHarness.onInboxAckResponse = undefined;+ credentialProxyHarness.onInboxAckObservationError = undefined;credentialProxyHarness.onProxyRequest = undefined;timelineRecorderHarness.pulls.splice(0);for (const dir of startupSweepDirs.splice(0)) rmSync(dir, { recursive: true, force: true });@@ -1440,10 +1455,25 @@ for await (const line of createInterface({ input: process.stdin })) {]]);expect(JSON.stringify(warnings)).not.toContain("private");expect(JSON.stringify(warnings)).not.toContain("Bearer");++ expect(credentialProxyHarness.onInboxAckObservationError).toBeTypeOf("function");+ credentialProxyHarness.onInboxAckObservationError?.({+ agentId: "agent-1",+ reason: "invalid_ack_shape",+ contentEncoding: "identity",+ body: "private ack body",+ });+ const ackWarnings = logger.calls.warn.filter(([, message]) => message === "inbox ack observation failed");+ expect(ackWarnings).toEqual([[+ "root",+ "inbox ack observation failed",+ [{ agentId: "agent-1", reason: "invalid_ack_shape", contentEncoding: "identity" }],+ ]]);+ expect(JSON.stringify(ackWarnings)).not.toContain("private");await daemon.stop();});- it("treats non-object or absent pull observation tokens as ownerless without model-seen updates", async () => {+ it("records pulled messages without model-seen and advances model-seen only after ack", async () => {const sockets: FakeSocket[] = [];const workingDirectoryBase = mkdtempSync(join(tmpdir(), "timeline-invalid-token-"));startupSweepDirs.push(workingDirectoryBase);@@ -1471,6 +1501,20 @@ for await (const line of createInterface({ input: process.stdin })) {expect(timelineRecorderHarness.pulls.map((pull) => pull.owner)).toEqual([null, null]);expect(recordModelSeen).not.toHaveBeenCalled();+ const ackToken = credentialProxyHarness.onInboxAckStart?.("bot_1");+ credentialProxyHarness.onInboxAckResponse?.(+ "bot_1",+ [{ channel: "/demo#1234/general", seq: 1 }],+ ackToken,+ );+ expect(recordModelSeen).toHaveBeenCalledOnce();+ expect(recordModelSeen).toHaveBeenCalledWith(+ "bot_1",+ [{ channel: "/demo#1234/general", seq: "#1" }],+ ackToken,+ );+ credentialProxyHarness.onInboxAckResponse?.("bot_1", [{ channel: "/demo#1234/general", seq: 2 }]);+ expect(recordModelSeen).toHaveBeenCalledOnce();await daemon.stop();});@@ -1742,13 +1786,15 @@ describe("createDaemon — logging", () => {const wakeAcked = (seq: number) => sockets[0].sent.map((frame) => JSON.parse(frame)).some((frame) => frame.type === "agent_wake_ack" && frame.launchId === `launch_${seq}` && frame.status === "ok",);- const observePull = (seq: number) => {+ const observeDelivery = (seq: number) => {const observationToken = credentialProxyHarness.onInboxPullStart?.("bot_working");credentialProxyHarness.onInboxPullResponse?.("bot_working",[{ channel, seq: `#${seq}` }],observationToken,);+ const ackToken = credentialProxyHarness.onInboxAckStart?.("bot_working");+ credentialProxyHarness.onInboxAckResponse?.("bot_working", [{ channel, seq }], ackToken);};try {@@ -1767,14 +1813,14 @@ describe("createDaemon — logging", () => {wake(1);await vi.waitFor(() => expect(starts).toHaveLength(1));await vi.waitFor(() => expect(wakeAcked(1)).toBe(true));- observePull(1);+ observeDelivery(1);for (let seq = 2; seq <= 6; seq++) wake(seq);await vi.waitFor(() => expect(wakeAcked(6)).toBe(true));expect(sends).toHaveLength(1);expect(sends[0]).toMatchObject({ sequence: 2 });- observePull(6);+ observeDelivery(6);await new Promise<void>((resolve) => setImmediate(resolve));expect(sends).toHaveLength(1);@@ -1784,7 +1830,7 @@ describe("createDaemon — logging", () => {expect(sends).toHaveLength(2);expect(sends[1]).toMatchObject({ sequence: 7 });- observePull(7);+ observeDelivery(7);await vi.waitFor(() => expect(sends).toHaveLength(3));expect(sends[2]).toMatchObject({ sequence: 8 });} finally {@@ -1840,13 +1886,15 @@ describe("createDaemon — logging", () => {&& frame.launchId === launchId&& frame.status === "ok",);- const observePull = (channel: string, seq: number) => {+ const observeDelivery = (channel: string, seq: number) => {const observationToken = credentialProxyHarness.onInboxPullStart?.("bot_replay");credentialProxyHarness.onInboxPullResponse?.("bot_replay",[{ channel, seq: `#${seq}` }],observationToken,);+ const ackToken = credentialProxyHarness.onInboxAckStart?.("bot_replay");+ credentialProxyHarness.onInboxAckResponse?.("bot_replay", [{ channel, seq }], ackToken);};try {@@ -1869,17 +1917,18 @@ describe("createDaemon — logging", () => {wake("/demo#1234/c2", 7, "c2");await vi.waitFor(() => expect(wakeAcked("c1")).toBe(true));await vi.waitFor(() => expect(wakeAcked("c2")).toBe(true));- observePull("/demo#1234/root", 1);+ observeDelivery("/demo#1234/root", 1);releaseEnroll();await vi.waitFor(() => expect(starts).toHaveLength(1));await vi.waitFor(() => expect(wakeAcked("root")).toBe(true));await vi.waitFor(() => expect(sends).toHaveLength(1));- // The pull proves only c1 reached the model. c2 must be re-admitted into- // the still-running session, using a fresh driver command identity.+ // The successful acknowledgement proves only c1 reached the model. c2+ // must be re-admitted into the still-running session, using a fresh+ // driver command identity.await new Promise((resolve) => setTimeout(resolve, 2));- observePull("/demo#1234/c1", 2);+ observeDelivery("/demo#1234/c1", 2);await vi.waitFor(() => expect(sends).toHaveLength(2));expect(starts[0]!.id).toContain(":admission:1");diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.tsindex a3508f84e..b31346822 100644--- a/src/daemon/src/daemon/createDaemon.ts+++ b/src/daemon/src/daemon/createDaemon.ts@@ -529,24 +529,31 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDaconst broker = new CredentialBroker({ upstreamBaseUrl: opts.serverUrl });const proxy = await startCredentialProxy(broker, {onInboxPullStart: (agentId) => ({- modelSeenGeneration: channelRef?.modelSeenGeneration(agentId),owner: managerRef?.timelineTurnOwner(agentId) ?? null,}),onInboxPullResponse: (agentId, messages, observationToken) => {const token = observationToken && typeof observationToken === "object"? observationToken as {- modelSeenGeneration?: unknown;owner?: ReturnType<AgentProcessManager["timelineTurnOwner"]>;}: null;timeline.recordInboxPull(agentId, token?.owner ?? null, messages);- if (typeof token?.modelSeenGeneration === "number") {- channelRef?.recordModelSeen(agentId, messages, token.modelSeenGeneration);- }},onInboxPullObservationError: ({ agentId, reason, contentEncoding }) => {log.warn("inbox pull timeline observation failed", { agentId, reason, contentEncoding });},+ onInboxAckStart: (agentId) => channelRef?.modelSeenGeneration(agentId),+ onInboxAckResponse: (agentId, applied, observationToken) => {+ if (typeof observationToken !== "number") return;+ channelRef?.recordModelSeen(+ agentId,+ applied.map((cursor) => ({ channel: cursor.channel, seq: `#${cursor.seq}` })),+ observationToken,+ );+ },+ onInboxAckObservationError: ({ agentId, reason, contentEncoding }) => {+ log.warn("inbox ack observation failed", { agentId, reason, contentEncoding });+ },onMessageReminderArm: (input) =>reminderSchedulerRef?.arm(input) ?? { armed: false, reason: "reminder_scheduler_unavailable" },// Bot audit log — Producer B (authoritative for `alook <sub>`). Firesdiff --git a/src/daemon/src/manager/wakeCoordinator.test.ts b/src/daemon/src/manager/wakeCoordinator.test.tsindex 90993b9a7..0e08e1693 100644--- a/src/daemon/src/manager/wakeCoordinator.test.ts+++ b/src/daemon/src/manager/wakeCoordinator.test.ts@@ -263,7 +263,7 @@ describe("WakeCoordinator", () => {expect(dispatch).toHaveBeenCalledTimes(2);});- it("suppresses an old wake when inbox pull arrived first", async () => {+ it("suppresses an old wake after model-seen confirmation", async () => {const coordinator = new WakeCoordinator();const dispatch = vi.fn(async () => {});coordinator.recordModelSeen("a1", [{ channel: "/s/c", seq: "#9" }], 0);@@ -274,7 +274,7 @@ describe("WakeCoordinator", () => {expect(dispatch).not.toHaveBeenCalled();});- it("admits a higher seq after pull coverage and idle", async () => {+ it("admits a higher seq after model-seen coverage and idle", async () => {const coordinator = new WakeCoordinator();const dispatch = vi.fn(async () => {});coordinator.recordModelSeen("a1", [{ channel: "/s/c", seq: "#5" }], 0);diff --git a/src/daemon/src/manager/wakeCoordinator.ts b/src/daemon/src/manager/wakeCoordinator.tsindex 31383fdf3..1cce0fd2c 100644--- a/src/daemon/src/manager/wakeCoordinator.ts+++ b/src/daemon/src/manager/wakeCoordinator.ts@@ -76,7 +76,7 @@ export class WakeCoordinator {// One unread notification may be outstanding at a time. An active runtime// still needs the first higher watermark delivered into its current// logical session; later watermarks coalesce behind that admission until- // the resulting inbox pull proves what the model actually saw.+ // the resulting successful inbox acknowledgement proves what the model saw.if (state.admitting || state.activeAdmission) {this.rememberReplacement(state, command);return { state: "suppressed", coveredSeq: seq };