Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

fix(daemon): keep slow inbox pulls replayable (#717)

Gustavo Yed1d1d0bddf2d
diff --git a/src/daemon/src/cli/index.test.ts b/src/daemon/src/cli/index.test.ts
index 9db1f7546..0a7bb4062 100644
--- a/src/daemon/src/cli/index.test.ts
+++ b/src/daemon/src/cli/index.test.ts
@@ -528,6 +528,92 @@ describe("inbox pull", () => {
expect(env.success.pulledAt).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{2}:\d{2}$/);
});
+ it("leaves a pull near the outer 10s wait unread so a fast retry can replay and acknowledge it", async () => {
+ const message = {
+ seq: "#7",
+ channel: "/s#0042/general",
+ sender: "@x",
+ content: { text: "replay me" },
+ time: "",
+ };
+ let readSeq = 0;
+ const ackSpy = vi.fn(async (req: { cursors: Array<{ channel: string; seq: number }> }) => {
+ readSeq = Math.max(readSeq, ...req.cursors.map((cursor) => cursor.seq));
+ return { ok: true, applied: req.cursors, failed: [] };
+ });
+ const nowSpy = vi.spyOn(performance, "now")
+ .mockReturnValueOnce(100)
+ .mockReturnValueOnce(9_900)
+ .mockReturnValueOnce(20_000)
+ .mockReturnValueOnce(20_001);
+ setApiForTesting(
+ stubApi({
+ inboxPull: async () => ({
+ messages: readSeq >= 7 ? [] : [message],
+ hasMore: false,
+ markedCount: 0,
+ }),
+ ack: ackSpy,
+ }),
+ );
+
+ try {
+ await main(["inbox", "pull"]);
+ expect(ackSpy).not.toHaveBeenCalled();
+ const slow = parseEnvelope(cap.lines()) as {
+ success: {
+ acked: number;
+ messages: unknown[];
+ ackDeferred?: { reason: string; thresholdMs: number; elapsedMs: number };
+ };
+ };
+ expect(slow.success.messages).toEqual([message]);
+ expect(slow.success.acked).toBe(0);
+ expect(slow.success.ackDeferred).toEqual({
+ reason: "slow_pull",
+ thresholdMs: 9_000,
+ elapsedMs: 9_800,
+ });
+
+ cap.lines().length = 0;
+ await main(["inbox", "pull"]);
+ } finally {
+ nowSpy.mockRestore();
+ }
+
+ const retry = parseEnvelope(cap.lines()) as {
+ success: { acked: number; messages: unknown[]; ackDeferred?: unknown };
+ };
+ expect(ackSpy).toHaveBeenCalledOnce();
+ expect(retry.success.messages).toEqual([message]);
+ expect(retry.success.acked).toBe(1);
+ expect(retry.success.ackDeferred).toBeUndefined();
+ });
+
+ it("does not report deferred acknowledgement for an empty slow pull", async () => {
+ const ackSpy = vi.fn();
+ const nowSpy = vi.spyOn(performance, "now")
+ .mockReturnValueOnce(0)
+ .mockReturnValueOnce(20_000);
+ setApiForTesting(stubApi({
+ inboxPull: async () => ({ messages: [], hasMore: false, markedCount: 0 }),
+ ack: ackSpy,
+ }));
+
+ try {
+ await main(["inbox", "pull"]);
+ } finally {
+ nowSpy.mockRestore();
+ }
+
+ const env = parseEnvelope(cap.lines()) as {
+ success: { acked: number; ackDeferred?: unknown };
+ };
+ expect(ackSpy).not.toHaveBeenCalled();
+ expect(env.success.acked).toBe(0);
+ expect(env.success.ackDeferred).toBeUndefined();
+ });
+
it("removes --no-ack from inbox pull help", async () => {
await main(["inbox", "pull", "-h"]);
expect(cap.lines().join("")).not.toContain("--no-ack");
diff --git a/src/daemon/src/cli/index.ts b/src/daemon/src/cli/index.ts
index 9d50e1c68..975375e24 100755
--- a/src/daemon/src/cli/index.ts
+++ b/src/daemon/src/cli/index.ts
@@ -209,6 +209,10 @@ function stripMalformedAlookHeredocTail(input: string): string {
const CLIENT_MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024;
const CLIENT_MAX_MESSAGE_BODY_BYTES = 1024;
+// The agent tool stops waiting after 10 seconds. Do not start an acknowledgement
+// once the pull has consumed the first nine, leaving the normal ack/output path
+// one second of headroom at that outer boundary.
+const INBOX_PULL_OUTPUT_DEADLINE_MS = 9_000;
/**
* Guess a content-type from a filename extension. Kept trivial — the server
@@ -597,12 +601,15 @@ async function cmdInboxPull(opts: Record<string, unknown>): Promise<unknown> {
const api = getApi();
const agent = agentId(opts);
const max = opts.max ? Number(opts.max) : undefined;
+ const pullStartedAt = performance.now();
const { messages, hasMore, markedCount } = await api.inboxPull({ agentId: agent, max });
+ const pullElapsedMs = performance.now() - pullStartedAt;
const pulledAt = nowLocalISO();
let acked = 0;
let ackError: string | undefined;
let failed: AckFailure[] | undefined;
+ let ackDeferred: { reason: "slow_pull"; thresholdMs: number; elapsedMs: number } | undefined;
if (messages.length > 0) {
const latest = new Map<string, Cursor>();
for (const m of messages) {
@@ -610,18 +617,22 @@ async function cmdInboxPull(opts: Record<string, unknown>): Promise<unknown> {
const cur = latest.get(m.channel);
if (!cur || seqN > cur.seq) latest.set(m.channel, { channel: m.channel, seq: seqN });
}
- try {
- const result = await api.ack({ agentId: agent, cursors: [...latest.values()] });
- acked = result.applied.length;
- failed = result.failed;
- } catch (err) {
- // Do NOT rethrow: the pull already succeeded, and if ack fails on a
- // single scope (e.g. a stale visibility mismatch) the whole envelope
- // would otherwise collapse to a bare error, wiping the messages the
- // agent needs. Surface the ack failure separately so the agent (or a
- // human debugging) sees BOTH the delivered messages AND that the
- // waterline didn't move.
- ackError = err instanceof Error ? err.message : String(err);
+ if (pullElapsedMs >= INBOX_PULL_OUTPUT_DEADLINE_MS) {
+ ackDeferred = {
+ reason: "slow_pull",
+ thresholdMs: INBOX_PULL_OUTPUT_DEADLINE_MS,
+ elapsedMs: Math.round(pullElapsedMs),
+ };
+ } else {
+ try {
+ const result = await api.ack({ agentId: agent, cursors: [...latest.values()] });
+ acked = result.applied.length;
+ failed = result.failed;
+ } catch (err) {
+ // Keep the successfully pulled messages visible even when the
+ // automatic acknowledgement fails.
+ ackError = err instanceof Error ? err.message : String(err);
+ }
}
}
@@ -632,6 +643,7 @@ async function cmdInboxPull(opts: Record<string, unknown>): Promise<unknown> {
pulledAt,
...(failed ? { failed } : {}),
...(ackError ? { ackError } : {}),
+ ...(ackDeferred ? { ackDeferred } : {}),
...(markedCount > 0
? {
markedReminder: `You have ${markedCount} marked ${markedCount === 1 ? "message" : "messages"}. Resolve ${markedCount === 1 ? "it" : "them"} before going dark unless blocked.`,
diff --git a/src/daemon/src/credentials/credentialProxy.test.ts b/src/daemon/src/credentials/credentialProxy.test.ts
index eb371e0ed..cfe680083 100644
--- a/src/daemon/src/credentials/credentialProxy.test.ts
+++ b/src/daemon/src/credentials/credentialProxy.test.ts
@@ -738,6 +738,175 @@ describe("startCredentialProxy (zero-trust end to end)", () => {
expect(onInboxPullObservationError).not.toHaveBeenCalled();
});
+ it("observes only server-applied cursors from a successful inbox ack", async () => {
+ const applied = [
+ { channel: "/s#0001/c", seq: 3 },
+ { channel: "/s#0001/private", seq: 8 },
+ ];
+ const body = JSON.stringify({
+ ok: false,
+ applied,
+ failed: [{ channel: "/s#0001/blocked", seq: 5, code: "forbidden", error: "forbidden" }],
+ });
+ const upstream = await startUpstream({ body });
+ upstreamClose = upstream.close;
+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });
+ const onInboxAckStart = vi.fn(() => ({ generation: 4 }));
+ const onInboxAckResponse = vi.fn();
+ const onInboxAckObservationError = vi.fn();
+ proxy = await startCredentialProxy(broker, {
+ onInboxAckStart,
+ onInboxAckResponse,
+ onInboxAckObservationError,
+ });
+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);
+
+ const response = await post(
+ proxy.url,
+ reg.voucher,
+ "/api/community/users/me/inbox/ack",
+ { "accept-encoding": "gzip, deflate" },
+ );
+
+ expect(response).toEqual({ status: 200, body });
+ expect(upstream.seen[0]?.acceptEncoding).toBe("identity");
+ expect(onInboxAckStart).toHaveBeenCalledWith("agent-1");
+ expect(onInboxAckResponse).toHaveBeenCalledOnce();
+ expect(onInboxAckResponse).toHaveBeenCalledWith("agent-1", applied, { generation: 4 });
+ expect(onInboxAckObservationError).not.toHaveBeenCalled();
+ });
+
+ it("does not observe a failed inbox ack response", async () => {
+ const body = JSON.stringify({ ok: true, applied: [{ channel: "/s#0001/c", seq: 3 }], failed: [] });
+ const upstream = await startUpstream({ status: 500, body });
+ upstreamClose = upstream.close;
+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });
+ const onInboxAckResponse = vi.fn();
+ const onInboxAckObservationError = vi.fn();
+ proxy = await startCredentialProxy(broker, { onInboxAckResponse, onInboxAckObservationError });
+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);
+
+ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");
+
+ expect(response).toEqual({ status: 500, body });
+ expect(onInboxAckResponse).not.toHaveBeenCalled();
+ expect(onInboxAckObservationError).not.toHaveBeenCalled();
+ });
+
+ it("forwards a malformed successful ack body without advancing model-seen", async () => {
+ const body = JSON.stringify({ ok: true, applied: [{ channel: "/s#0001/c", seq: "3" }] });
+ const upstream = await startUpstream({ body });
+ upstreamClose = upstream.close;
+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });
+ const onInboxAckResponse = vi.fn();
+ const onInboxAckObservationError = vi.fn();
+ proxy = await startCredentialProxy(broker, { onInboxAckResponse, onInboxAckObservationError });
+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);
+
+ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");
+
+ expect(response).toEqual({ status: 200, body });
+ expect(onInboxAckResponse).not.toHaveBeenCalled();
+ expect(onInboxAckObservationError).toHaveBeenCalledWith({
+ agentId: "agent-1",
+ reason: "invalid_ack_shape",
+ contentEncoding: "identity",
+ });
+ });
+
+ it("forwards inbox ack when the observation start hook throws", async () => {
+ const body = JSON.stringify({ ok: true, applied: [], failed: [] });
+ const upstream = await startUpstream({ body });
+ upstreamClose = upstream.close;
+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });
+ const onInboxAckObservationError = vi.fn();
+ proxy = await startCredentialProxy(broker, {
+ onInboxAckStart: () => {
+ throw new Error("start observer failed");
+ },
+ onInboxAckResponse: vi.fn(),
+ onInboxAckObservationError,
+ });
+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);
+
+ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");
+
+ expect(response).toEqual({ status: 200, body });
+ expect(onInboxAckObservationError).toHaveBeenCalledWith({
+ agentId: "agent-1",
+ reason: "observer_failed",
+ contentEncoding: "identity",
+ });
+ });
+
+ it("forwards an encoded inbox ack without attempting to parse it", async () => {
+ const json = JSON.stringify({ ok: true, applied: [{ channel: "/s#0001/c", seq: 3 }], failed: [] });
+ const upstream = await startUpstream({
+ headers: { "content-encoding": "gzip" },
+ body: gzipSync(json),
+ });
+ upstreamClose = upstream.close;
+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });
+ const onInboxAckResponse = vi.fn();
+ const onInboxAckObservationError = vi.fn();
+ proxy = await startCredentialProxy(broker, { onInboxAckResponse, onInboxAckObservationError });
+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);
+
+ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");
+
+ expect(response).toEqual({ status: 200, body: json });
+ expect(onInboxAckResponse).not.toHaveBeenCalled();
+ expect(onInboxAckObservationError).toHaveBeenCalledWith({
+ agentId: "agent-1",
+ reason: "unexpected_content_encoding",
+ contentEncoding: "gzip",
+ });
+ });
+
+ it("keeps forwarding invalid ack JSON when the error observer also throws", async () => {
+ const body = "not-json";
+ const upstream = await startUpstream({ body });
+ upstreamClose = upstream.close;
+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });
+ const onInboxAckResponse = vi.fn();
+ proxy = await startCredentialProxy(broker, {
+ onInboxAckResponse,
+ onInboxAckObservationError: () => {
+ throw new Error("error observer failed");
+ },
+ });
+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);
+
+ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");
+
+ expect(response).toEqual({ status: 200, body });
+ expect(onInboxAckResponse).not.toHaveBeenCalled();
+ });
+
+ it("keeps forwarding a valid ack when the response observer throws", async () => {
+ const body = JSON.stringify({ ok: true, applied: [{ channel: "/s#0001/c", seq: 3 }], failed: [] });
+ const upstream = await startUpstream({ body });
+ upstreamClose = upstream.close;
+ const broker = new CredentialBroker({ upstreamBaseUrl: upstream.url });
+ const onInboxAckObservationError = vi.fn();
+ proxy = await startCredentialProxy(broker, {
+ onInboxAckResponse: () => {
+ throw new Error("ack observer failed");
+ },
+ onInboxAckObservationError,
+ });
+ const reg = broker.mint("agent-1", "l", ["read"], REAL_KEY);
+
+ const response = await post(proxy.url, reg.voucher, "/api/community/users/me/inbox/ack");
+
+ expect(response).toEqual({ status: 200, body });
+ expect(onInboxAckObservationError).toHaveBeenCalledWith({
+ agentId: "agent-1",
+ reason: "observer_failed",
+ contentEncoding: "identity",
+ });
+ });
+
it("preserves compression negotiation for non-pull and wrong-method lookalike routes", async () => {
const upstream = await startUpstream({ body: JSON.stringify({ messages: [{ seq: "#1" }] }) });
upstreamClose = upstream.close;
diff --git a/src/daemon/src/credentials/credentialProxy.ts b/src/daemon/src/credentials/credentialProxy.ts
index f34a564cc..ab866dc59 100644
--- a/src/daemon/src/credentials/credentialProxy.ts
+++ b/src/daemon/src/credentials/credentialProxy.ts
@@ -44,7 +44,7 @@ import * as https from "https";
import * as os from "os";
import * as path from "path";
import { URL } from "url";
-import { formatRef, parseRef, type Message } from "../server/contract.js";
+import { formatRef, parseRef, type Cursor, type Message } from "../server/contract.js";
import { parseNameAndTag } from "@alook/shared/lib/discriminator";
export const LOCAL_MESSAGE_REMINDER_PATH = "/__alook/local/message-reminder";
@@ -318,6 +318,9 @@ export interface CredentialProxyOptions {
onInboxPullStart?: (agentId: string) => unknown;
onInboxPullResponse?: (agentId: string, messages: Message[], observationToken?: unknown) => void;
onInboxPullObservationError?: (failure: InboxPullObservationFailure) => void;
+ onInboxAckStart?: (agentId: string) => unknown;
+ onInboxAckResponse?: (agentId: string, applied: Cursor[], observationToken?: unknown) => void;
+ onInboxAckObservationError?: (failure: InboxAckObservationFailure) => void;
/**
* Called once per successfully-authorized upstream proxy request, BEFORE the
* upstream is dispatched. Fires only on `verdict.ok === true` — never on
@@ -364,6 +367,18 @@ export interface InboxPullObservationFailure {
contentEncoding: InboxPullContentEncoding;
}
+export type InboxAckObservationFailureReason =
+ | "unexpected_content_encoding"
+ | "invalid_json"
+ | "invalid_ack_shape"
+ | "observer_failed";
+
+export interface InboxAckObservationFailure {
+ agentId: string;
+ reason: InboxAckObservationFailureReason;
+ contentEncoding: InboxPullContentEncoding;
+}
+
export interface RunningProxy {
url: string;
port: number;
@@ -510,6 +525,7 @@ export async function startCredentialProxy(
const upstreamClient = upstream.protocol === "https:" ? https : http;
const onPull = options.onInboxPullResponse;
+ const onAck = options.onInboxAckResponse;
const onProxyRequest = options.onProxyRequest;
const reportInboxPullObservationError = (failure: InboxPullObservationFailure): void => {
try {
@@ -518,6 +534,13 @@ export async function startCredentialProxy(
// Observational callback — never disrupt the proxy response.
}
};
+ const reportInboxAckObservationError = (failure: InboxAckObservationFailure): void => {
+ try {
+ options.onInboxAckObservationError?.(failure);
+ } catch {
+ return;
+ }
+ };
const server = http.createServer((req, res) => {
const pathname = new URL(req.url ?? "/", "http://placeholder").pathname;
@@ -581,8 +604,11 @@ export async function startCredentialProxy(
// `/api/inboxPull` verb is deleted (flat-delete step). inboxSnapshot is
// deliberately EXCLUDED (peek, no `{ messages }` to record).
const isInboxPull = req.method === "POST" && pathname === "/api/community/users/me/inbox/pull";
+ const isInboxAck = req.method === "POST" && pathname === "/api/community/users/me/inbox/ack";
const shouldObserveInboxPull = Boolean(isInboxPull && onPull);
+ const shouldObserveInboxAck = Boolean(isInboxAck && onAck);
let inboxPullObservationToken: unknown;
+ let inboxAckObservationToken: unknown;
if (shouldObserveInboxPull) {
try {
inboxPullObservationToken = options.onInboxPullStart?.(reg.agentId);
@@ -594,6 +620,17 @@ export async function startCredentialProxy(
});
}
}
+ if (shouldObserveInboxAck) {
+ try {
+ inboxAckObservationToken = options.onInboxAckStart?.(reg.agentId);
+ } catch {
+ reportInboxAckObservationError({
+ agentId: reg.agentId,
+ reason: "observer_failed",
+ contentEncoding: "identity",
+ });
+ }
+ }
if (onProxyRequest) {
try {
@@ -616,7 +653,7 @@ export async function startCredentialProxy(
outHeaders[broker.headerNames.agentId.toLowerCase()] = reg.agentId;
outHeaders[broker.headerNames.client.toLowerCase()] = broker.clientLabel;
outHeaders[broker.headerNames.capabilities.toLowerCase()] = [...reg.capabilities].join(",");
- if (isInboxPull) outHeaders["accept-encoding"] = "identity";
+ if (isInboxPull || isInboxAck) outHeaders["accept-encoding"] = "identity";
// `responded` guards only against writing a second response to the
// DOWNSTREAM client (writeHead/end can't fire twice) — it does NOT gate
@@ -654,53 +691,97 @@ export async function startCredentialProxy(
res_.on("error", destroyResIfIncomplete);
res_.on("close", destroyResIfIncomplete);
if (
- shouldObserveInboxPull
+ (shouldObserveInboxPull || shouldObserveInboxAck)
&& res_.statusCode !== undefined
&& res_.statusCode >= 200
&& res_.statusCode < 300
) {
- // Buffer the inboxPull response to surface pulled messages to the daemon.
+ // Buffer observed inbox responses so the daemon can track pull and ack boundaries.
const chunks: Buffer[] = [];
res_.on("data", (chunk: Buffer) => chunks.push(chunk));
res_.on("end", () => {
const body = Buffer.concat(chunks);
const contentEncoding = normalizeContentEncoding(res_.headers["content-encoding"]);
if (contentEncoding !== "identity") {
- reportInboxPullObservationError({
- agentId: reg.agentId,
- reason: "unexpected_content_encoding",
- contentEncoding,
- });
+ if (shouldObserveInboxPull) {
+ reportInboxPullObservationError({
+ agentId: reg.agentId,
+ reason: "unexpected_content_encoding",
+ contentEncoding,
+ });
+ } else {
+ reportInboxAckObservationError({
+ agentId: reg.agentId,
+ reason: "unexpected_content_encoding",
+ contentEncoding,
+ });
+ }
} else {
let parsed: unknown;
try {
parsed = JSON.parse(body.toString("utf8"));
} catch {
- reportInboxPullObservationError({
- agentId: reg.agentId,
- reason: "invalid_json",
- contentEncoding,
- });
+ if (shouldObserveInboxPull) {
+ reportInboxPullObservationError({ agentId: reg.agentId, reason: "invalid_json", contentEncoding });
+ } else {
+ reportInboxAckObservationError({ agentId: reg.agentId, reason: "invalid_json", contentEncoding });
+ }
}
if (parsed !== undefined) {
- if (
+ if (shouldObserveInboxPull) {
+ if (
+ !parsed
+ || typeof parsed !== "object"
+ || Array.isArray(parsed)
+ || !Array.isArray((parsed as { messages?: unknown }).messages)
+ ) {
+ reportInboxPullObservationError({
+ agentId: reg.agentId,
+ reason: "invalid_inbox_shape",
+ contentEncoding,
+ });
+ } else {
+ const messages = (parsed as { messages: Message[] }).messages;
+ if (messages.length > 0) {
+ try {
+ onPull!(reg.agentId, messages, inboxPullObservationToken);
+ } catch {
+ reportInboxPullObservationError({
+ agentId: reg.agentId,
+ reason: "observer_failed",
+ contentEncoding,
+ });
+ }
+ }
+ }
+ } else if (
!parsed
|| typeof parsed !== "object"
|| Array.isArray(parsed)
- || !Array.isArray((parsed as { messages?: unknown }).messages)
+ || !Array.isArray((parsed as { applied?: unknown }).applied)
+ || !(parsed as { applied: unknown[] }).applied.every((cursor) => (
+ Boolean(cursor)
+ && typeof cursor === "object"
+ && !Array.isArray(cursor)
+ && typeof (cursor as { channel?: unknown }).channel === "string"
+ && (cursor as { channel: string }).channel.length > 0
+ && typeof (cursor as { seq?: unknown }).seq === "number"
+ && Number.isSafeInteger((cursor as { seq: number }).seq)
+ && (cursor as { seq: number }).seq > 0
+ ))
) {
- reportInboxPullObservationError({
+ reportInboxAckObservationError({
agentId: reg.agentId,
- reason: "invalid_inbox_shape",
+ reason: "invalid_ack_shape",
contentEncoding,
});
} else {
- const messages = (parsed as { messages: Message[] }).messages;
- if (messages.length > 0) {
+ const applied = (parsed as { applied: Cursor[] }).applied;
+ if (applied.length > 0) {
try {
- onPull!(reg.agentId, messages, inboxPullObservationToken);
+ onAck!(reg.agentId, applied, inboxAckObservationToken);
} catch {
- reportInboxPullObservationError({
+ reportInboxAckObservationError({
agentId: reg.agentId,
reason: "observer_failed",
contentEncoding,
diff --git a/src/daemon/src/daemon/createDaemon.test.ts b/src/daemon/src/daemon/createDaemon.test.ts
index 2c7ceecc0..fa263dd56 100644
--- a/src/daemon/src/daemon/createDaemon.test.ts
+++ b/src/daemon/src/daemon/createDaemon.test.ts
@@ -58,6 +58,11 @@ const credentialProxyHarness = vi.hoisted(() => ({
((agentId: string, messages: unknown[], observationToken?: unknown) => void) | undefined
),
onInboxPullObservationError: undefined as ((failure: Record<string, unknown>) => void) | undefined,
+ onInboxAckStart: undefined as ((agentId: string) => unknown) | undefined,
+ onInboxAckResponse: undefined as (
+ ((agentId: string, applied: Array<{ channel: string; seq: number }>, observationToken?: unknown) => void) | undefined
+ ),
+ onInboxAckObservationError: undefined as ((failure: Record<string, unknown>) => void) | undefined,
onProxyRequest: undefined as ((
agentId: string,
method: string,
@@ -102,6 +107,13 @@ vi.mock("../credentials/index.js", async (importOriginal) => {
credentialProxyHarness.onInboxPullObservationError = args[1]?.onInboxPullObservationError as
| ((failure: Record<string, unknown>) => void)
| undefined;
+ credentialProxyHarness.onInboxAckStart = args[1]?.onInboxAckStart;
+ credentialProxyHarness.onInboxAckResponse = args[1]?.onInboxAckResponse as
+ | ((agentId: string, applied: Array<{ channel: string; seq: number }>, observationToken?: unknown) => void)
+ | undefined;
+ credentialProxyHarness.onInboxAckObservationError = args[1]?.onInboxAckObservationError as
+ | ((failure: Record<string, unknown>) => void)
+ | undefined;
credentialProxyHarness.onProxyRequest = args[1]?.onProxyRequest;
return actual.startCredentialProxy(...args);
},
@@ -116,6 +128,9 @@ afterEach(() => {
credentialProxyHarness.onInboxPullStart = undefined;
credentialProxyHarness.onInboxPullResponse = undefined;
credentialProxyHarness.onInboxPullObservationError = undefined;
+ credentialProxyHarness.onInboxAckStart = undefined;
+ credentialProxyHarness.onInboxAckResponse = undefined;
+ credentialProxyHarness.onInboxAckObservationError = undefined;
credentialProxyHarness.onProxyRequest = undefined;
timelineRecorderHarness.pulls.splice(0);
for (const dir of startupSweepDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
@@ -1440,10 +1455,25 @@ for await (const line of createInterface({ input: process.stdin })) {
]]);
expect(JSON.stringify(warnings)).not.toContain("private");
expect(JSON.stringify(warnings)).not.toContain("Bearer");
+
+ expect(credentialProxyHarness.onInboxAckObservationError).toBeTypeOf("function");
+ credentialProxyHarness.onInboxAckObservationError?.({
+ agentId: "agent-1",
+ reason: "invalid_ack_shape",
+ contentEncoding: "identity",
+ body: "private ack body",
+ });
+ const ackWarnings = logger.calls.warn.filter(([, message]) => message === "inbox ack observation failed");
+ expect(ackWarnings).toEqual([[
+ "root",
+ "inbox ack observation failed",
+ [{ agentId: "agent-1", reason: "invalid_ack_shape", contentEncoding: "identity" }],
+ ]]);
+ expect(JSON.stringify(ackWarnings)).not.toContain("private");
await daemon.stop();
});
- it("treats non-object or absent pull observation tokens as ownerless without model-seen updates", async () => {
+ it("records pulled messages without model-seen and advances model-seen only after ack", async () => {
const sockets: FakeSocket[] = [];
const workingDirectoryBase = mkdtempSync(join(tmpdir(), "timeline-invalid-token-"));
startupSweepDirs.push(workingDirectoryBase);
@@ -1471,6 +1501,20 @@ for await (const line of createInterface({ input: process.stdin })) {
expect(timelineRecorderHarness.pulls.map((pull) => pull.owner)).toEqual([null, null]);
expect(recordModelSeen).not.toHaveBeenCalled();
+ const ackToken = credentialProxyHarness.onInboxAckStart?.("bot_1");
+ credentialProxyHarness.onInboxAckResponse?.(
+ "bot_1",
+ [{ channel: "/demo#1234/general", seq: 1 }],
+ ackToken,
+ );
+ expect(recordModelSeen).toHaveBeenCalledOnce();
+ expect(recordModelSeen).toHaveBeenCalledWith(
+ "bot_1",
+ [{ channel: "/demo#1234/general", seq: "#1" }],
+ ackToken,
+ );
+ credentialProxyHarness.onInboxAckResponse?.("bot_1", [{ channel: "/demo#1234/general", seq: 2 }]);
+ expect(recordModelSeen).toHaveBeenCalledOnce();
await daemon.stop();
});
@@ -1742,13 +1786,15 @@ describe("createDaemon — logging", () => {
const wakeAcked = (seq: number) => sockets[0].sent.map((frame) => JSON.parse(frame)).some(
(frame) => frame.type === "agent_wake_ack" && frame.launchId === `launch_${seq}` && frame.status === "ok",
);
- const observePull = (seq: number) => {
+ const observeDelivery = (seq: number) => {
const observationToken = credentialProxyHarness.onInboxPullStart?.("bot_working");
credentialProxyHarness.onInboxPullResponse?.(
"bot_working",
[{ channel, seq: `#${seq}` }],
observationToken,
);
+ const ackToken = credentialProxyHarness.onInboxAckStart?.("bot_working");
+ credentialProxyHarness.onInboxAckResponse?.("bot_working", [{ channel, seq }], ackToken);
};
try {
@@ -1767,14 +1813,14 @@ describe("createDaemon — logging", () => {
wake(1);
await vi.waitFor(() => expect(starts).toHaveLength(1));
await vi.waitFor(() => expect(wakeAcked(1)).toBe(true));
- observePull(1);
+ observeDelivery(1);
for (let seq = 2; seq <= 6; seq++) wake(seq);
await vi.waitFor(() => expect(wakeAcked(6)).toBe(true));
expect(sends).toHaveLength(1);
expect(sends[0]).toMatchObject({ sequence: 2 });
- observePull(6);
+ observeDelivery(6);
await new Promise<void>((resolve) => setImmediate(resolve));
expect(sends).toHaveLength(1);
@@ -1784,7 +1830,7 @@ describe("createDaemon — logging", () => {
expect(sends).toHaveLength(2);
expect(sends[1]).toMatchObject({ sequence: 7 });
- observePull(7);
+ observeDelivery(7);
await vi.waitFor(() => expect(sends).toHaveLength(3));
expect(sends[2]).toMatchObject({ sequence: 8 });
} finally {
@@ -1840,13 +1886,15 @@ describe("createDaemon — logging", () => {
&& frame.launchId === launchId
&& frame.status === "ok",
);
- const observePull = (channel: string, seq: number) => {
+ const observeDelivery = (channel: string, seq: number) => {
const observationToken = credentialProxyHarness.onInboxPullStart?.("bot_replay");
credentialProxyHarness.onInboxPullResponse?.(
"bot_replay",
[{ channel, seq: `#${seq}` }],
observationToken,
);
+ const ackToken = credentialProxyHarness.onInboxAckStart?.("bot_replay");
+ credentialProxyHarness.onInboxAckResponse?.("bot_replay", [{ channel, seq }], ackToken);
};
try {
@@ -1869,17 +1917,18 @@ describe("createDaemon — logging", () => {
wake("/demo#1234/c2", 7, "c2");
await vi.waitFor(() => expect(wakeAcked("c1")).toBe(true));
await vi.waitFor(() => expect(wakeAcked("c2")).toBe(true));
- observePull("/demo#1234/root", 1);
+ observeDelivery("/demo#1234/root", 1);
releaseEnroll();
await vi.waitFor(() => expect(starts).toHaveLength(1));
await vi.waitFor(() => expect(wakeAcked("root")).toBe(true));
await vi.waitFor(() => expect(sends).toHaveLength(1));
- // The pull proves only c1 reached the model. c2 must be re-admitted into
- // the still-running session, using a fresh driver command identity.
+ // The successful acknowledgement proves only c1 reached the model. c2
+ // must be re-admitted into the still-running session, using a fresh
+ // driver command identity.
await new Promise((resolve) => setTimeout(resolve, 2));
- observePull("/demo#1234/c1", 2);
+ observeDelivery("/demo#1234/c1", 2);
await vi.waitFor(() => expect(sends).toHaveLength(2));
expect(starts[0]!.id).toContain(":admission:1");
diff --git a/src/daemon/src/daemon/createDaemon.ts b/src/daemon/src/daemon/createDaemon.ts
index a3508f84e..b31346822 100644
--- a/src/daemon/src/daemon/createDaemon.ts
+++ b/src/daemon/src/daemon/createDaemon.ts
@@ -529,24 +529,31 @@ export async function createDaemon(opts: CreateDaemonOptions): Promise<RunningDa
const broker = new CredentialBroker({ upstreamBaseUrl: opts.serverUrl });
const proxy = await startCredentialProxy(broker, {
onInboxPullStart: (agentId) => ({
- modelSeenGeneration: channelRef?.modelSeenGeneration(agentId),
owner: managerRef?.timelineTurnOwner(agentId) ?? null,
}),
onInboxPullResponse: (agentId, messages, observationToken) => {
const token = observationToken && typeof observationToken === "object"
? observationToken as {
- modelSeenGeneration?: unknown;
owner?: ReturnType<AgentProcessManager["timelineTurnOwner"]>;
}
: null;
timeline.recordInboxPull(agentId, token?.owner ?? null, messages);
- if (typeof token?.modelSeenGeneration === "number") {
- channelRef?.recordModelSeen(agentId, messages, token.modelSeenGeneration);
- }
},
onInboxPullObservationError: ({ agentId, reason, contentEncoding }) => {
log.warn("inbox pull timeline observation failed", { agentId, reason, contentEncoding });
},
+ onInboxAckStart: (agentId) => channelRef?.modelSeenGeneration(agentId),
+ onInboxAckResponse: (agentId, applied, observationToken) => {
+ if (typeof observationToken !== "number") return;
+ channelRef?.recordModelSeen(
+ agentId,
+ applied.map((cursor) => ({ channel: cursor.channel, seq: `#${cursor.seq}` })),
+ observationToken,
+ );
+ },
+ onInboxAckObservationError: ({ agentId, reason, contentEncoding }) => {
+ log.warn("inbox ack observation failed", { agentId, reason, contentEncoding });
+ },
onMessageReminderArm: (input) =>
reminderSchedulerRef?.arm(input) ?? { armed: false, reason: "reminder_scheduler_unavailable" },
// Bot audit log — Producer B (authoritative for `alook <sub>`). Fires
diff --git a/src/daemon/src/manager/wakeCoordinator.test.ts b/src/daemon/src/manager/wakeCoordinator.test.ts
index 90993b9a7..0e08e1693 100644
--- a/src/daemon/src/manager/wakeCoordinator.test.ts
+++ b/src/daemon/src/manager/wakeCoordinator.test.ts
@@ -263,7 +263,7 @@ describe("WakeCoordinator", () => {
expect(dispatch).toHaveBeenCalledTimes(2);
});
- it("suppresses an old wake when inbox pull arrived first", async () => {
+ it("suppresses an old wake after model-seen confirmation", async () => {
const coordinator = new WakeCoordinator();
const dispatch = vi.fn(async () => {});
coordinator.recordModelSeen("a1", [{ channel: "/s/c", seq: "#9" }], 0);
@@ -274,7 +274,7 @@ describe("WakeCoordinator", () => {
expect(dispatch).not.toHaveBeenCalled();
});
- it("admits a higher seq after pull coverage and idle", async () => {
+ it("admits a higher seq after model-seen coverage and idle", async () => {
const coordinator = new WakeCoordinator();
const dispatch = vi.fn(async () => {});
coordinator.recordModelSeen("a1", [{ channel: "/s/c", seq: "#5" }], 0);
diff --git a/src/daemon/src/manager/wakeCoordinator.ts b/src/daemon/src/manager/wakeCoordinator.ts
index 31383fdf3..1cce0fd2c 100644
--- a/src/daemon/src/manager/wakeCoordinator.ts
+++ b/src/daemon/src/manager/wakeCoordinator.ts
@@ -76,7 +76,7 @@ export class WakeCoordinator {
// One unread notification may be outstanding at a time. An active runtime
// still needs the first higher watermark delivered into its current
// logical session; later watermarks coalesce behind that admission until
- // the resulting inbox pull proves what the model actually saw.
+ // the resulting successful inbox acknowledgement proves what the model saw.
if (state.admitting || state.activeAdmission) {
this.rememberReplacement(state, command);
return { state: "suppressed", coveredSeq: seq };