Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Record reply-detection failures instead of discarding them (#2475)

Regan Bellcd48d3fe1e8a
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts
index c76efcbc0..687b577a3 100644
--- a/src/core/orchestrator.ts
+++ b/src/core/orchestrator.ts
@@ -92,6 +92,7 @@ import {
} from "../auth/capability-token.ts";
import type {
GapWork,
+ HarnessDetectResult,
HarnessLlmRequestRecord,
HarnessTurnInput,
HarnessTurnResult,
@@ -2348,21 +2349,35 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
) {
const detectHistory = filterHistory((await historyWindow()).entries.slice(-DETECT_HISTORY_TAIL));
const detectStart = Date.now();
- const decision = await deps.harness.models.shouldRespond({
- session,
- message: input.text,
- recentContext: input.detectContext ?? "",
- ...(input.detectOpener ? { threadOpener: input.detectOpener } : {}),
- systemPrompt: resolution.systemPrompt,
- ...(input.gatewayContext?.reactionGuidance
- ? { reactionGuidance: input.gatewayContext.reactionGuidance }
- : {}),
- history: detectHistory,
- recordModelCall: (rec) => {
- deps.modelGateway.recordCall({ at: Date.now(), scopeLabel: scopeId, ...rec });
- void deps.budget?.record(actor.id, estimateCostUsd(rec.inputTokens));
- },
- });
+ const decision = await deps.harness.models
+ .shouldRespond({
+ session,
+ message: input.text,
+ recentContext: input.detectContext ?? "",
+ ...(input.detectOpener ? { threadOpener: input.detectOpener } : {}),
+ systemPrompt: resolution.systemPrompt,
+ ...(input.gatewayContext?.reactionGuidance
+ ? { reactionGuidance: input.gatewayContext.reactionGuidance }
+ : {}),
+ history: detectHistory,
+ recordModelCall: (rec) => {
+ deps.modelGateway.recordCall({ at: Date.now(), scopeLabel: scopeId, ...rec });
+ void deps.budget?.record(actor.id, estimateCostUsd(rec.inputTokens));
+ },
+ })
+ .catch((e: unknown): HarnessDetectResult => {
+ deps.errors?.record(
+ {
+ category: "turn",
+ code: "detection_failed",
+ message: errMessage(e),
+ scopeLabel: scopeId,
+ sessionId: session.id,
+ },
+ e,
+ );
+ return { respond: false, reason: `detection failed: ${errMessage(e)}` };
+ });
detectMs = Date.now() - detectStart;
if (!decision.respond) {
const reactions = decision.reactions?.length ? decision.reactions : undefined;
diff --git a/src/harness/claude-harness.ts b/src/harness/claude-harness.ts
index df6485d88..d6677c276 100644
--- a/src/harness/claude-harness.ts
+++ b/src/harness/claude-harness.ts
@@ -858,19 +858,14 @@ export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness {
},
resetSession: () => {},
async shouldRespond(detect) {
- try {
- const out = await single(
- buildDetectionPrompt(detect.reactionGuidance),
- renderDetectPrompt(detect),
- undefined,
- { recordModelCall: detect.recordModelCall },
- judgeModelId,
- );
- return parseDetectVerdict((out ?? "").trim(), Boolean(detect.reactionGuidance?.trim()));
- } catch (error) {
- swallow("claude: detect", error);
- return { respond: false };
- }
+ const out = await single(
+ buildDetectionPrompt(detect.reactionGuidance),
+ renderDetectPrompt(detect),
+ undefined,
+ { recordModelCall: detect.recordModelCall },
+ judgeModelId,
+ );
+ return parseDetectVerdict((out ?? "").trim(), Boolean(detect.reactionGuidance?.trim()));
},
async compactHistory(input) {
const model = getRequiredModel(resolveModelId());
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index 4683a59ac..bd25ffc62 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -2406,25 +2406,21 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
},
async shouldRespond(detect: HarnessDetectInput): Promise<HarnessDetectResult> {
- try {
- const modelId = detectModelId();
- const model = requiredModel(modelId);
- const providerKeys = await resolveProviderKeys();
- if (!keyForModel(providerKeys, model)) return { respond: true };
- const detectSystemPrompt = buildDetectionPrompt(detect.reactionGuidance);
- const prompt = renderDetectPrompt(detect);
- detect.recordModelCall({
- model: modelId,
- inputTokens: countTokens(detectSystemPrompt) + countTokens(prompt),
- entryCount: detect.history.length,
- });
- const out = (
- (await oneShot("pi-detect", model, providerKeys, detectSystemPrompt, prompt, { modelGateway })) ?? ""
- ).trim();
- return parseDetectVerdict(out, Boolean(detect.reactionGuidance?.trim()));
- } catch {
- return { respond: false };
- }
+ const modelId = detectModelId();
+ const model = requiredModel(modelId);
+ const providerKeys = await resolveProviderKeys();
+ if (!keyForModel(providerKeys, model)) return { respond: true };
+ const detectSystemPrompt = buildDetectionPrompt(detect.reactionGuidance);
+ const prompt = renderDetectPrompt(detect);
+ detect.recordModelCall({
+ model: modelId,
+ inputTokens: countTokens(detectSystemPrompt) + countTokens(prompt),
+ entryCount: detect.history.length,
+ });
+ const out = (
+ (await oneShot("pi-detect", model, providerKeys, detectSystemPrompt, prompt, { modelGateway })) ?? ""
+ ).trim();
+ return parseDetectVerdict(out, Boolean(detect.reactionGuidance?.trim()));
},
async compactHistory(input: HarnessCompactInput): Promise<string> {
diff --git a/test/reply-detection-failure.test.ts b/test/reply-detection-failure.test.ts
new file mode 100644
index 000000000..a7cb17978
--- /dev/null
+++ b/test/reply-detection-failure.test.ts
@@ -0,0 +1,96 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createOrchestrator } from "../src/core/orchestrator.ts";
+import { createIdentityService } from "../src/identity/identity-service.ts";
+import { createMemoryConfigStore } from "../src/resolution/config-store.ts";
+import { createAclStore } from "../src/acl/acl-store.ts";
+import { createResolutionService } from "../src/resolution/resolution-service.ts";
+import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";
+import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";
+import { createMemoryService } from "../src/memory/memory-service.ts";
+import { createModelGateway } from "../src/model/model-gateway.ts";
+import { createAuditLog } from "../src/audit/audit-log.ts";
+import { createErrorLog } from "../src/admin/error-log.ts";
+import { createRateLimiter } from "../src/ratelimit/rate-limiter.ts";
+import { createMockHarness } from "../src/harness/mock-harness.ts";
+import { createDeployStore } from "../src/deploy/deploy-store.ts";
+import { createDockerDeployProvider } from "../src/deploy/docker-deploy-provider.ts";
+import { createDeployService } from "../src/deploy/deploy-service.ts";
+import { createMemoryFileArtifactStore } from "../src/files/file-artifact-store.ts";
+import { createMemoryDurableByteStore } from "../src/files/durable-byte-store.ts";
+import type { Harness } from "../src/harness/harness.ts";
+import type { Sandbox } from "../src/sandbox/sandbox.ts";
+import type { Conversation, Principal } from "../src/types.ts";
+
+const actor: Principal = { id: "U1", type: "internal" };
+
+test("an ambient message whose reply detection fails stays silent and leaves the failure in the error and audit logs", async () => {
+ const base = createMockHarness();
+ let ran = false;
+ const harness: Harness = {
+ ...base,
+ turns: {
+ ...base.turns,
+ runTurn: async () => {
+ ran = true;
+ return { reply: "spurious" };
+ },
+ },
+ models: {
+ ...base.models,
+ shouldRespond: async () => {
+ throw new Error("401 invalid x-api-key");
+ },
+ },
+ };
+ const config = createMemoryConfigStore("default-org");
+ const acl = createAclStore();
+ const auditLog = createAuditLog();
+ const errors = createErrorLog();
+ const workspace = createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "rdf-")));
+ const orch = createOrchestrator({
+ identity: createIdentityService(),
+ resolution: createResolutionService("default-org", config, acl),
+ sessions: createMemorySessionStore(),
+ workspace,
+ files: createMemoryFileArtifactStore(createMemoryDurableByteStore()),
+ sandbox: {
+ profile: { backend: "fake", writablePersistence: "snapshot_to_workspace", processSessions: false },
+ } as Sandbox,
+ modelGateway: createModelGateway(),
+ auditLog,
+ errors,
+ rateLimiter: createRateLimiter({ maxPerWindow: 1000, windowMs: 60_000 }),
+ harness,
+ memory: createMemoryService(workspace),
+ deploy: createDeployService({
+ deployStore: createDeployStore(),
+ provider: createDockerDeployProvider(),
+ deployDir: join(tmpdir(), "rdf-deploy"),
+ auditLog,
+ acl,
+ }),
+ acl,
+ });
+
+ const result = await orch.handleTurn({
+ surface: "slack",
+ actor,
+ conversation: { kind: "channel", threadRef: "ch:C1:200.1", channelRef: "C1", audience: [actor] } as Conversation,
+ origin: { kind: "ambient" },
+ text: "anyone know why the deploy is stuck?",
+ });
+
+ assert.equal(result.status, "silent");
+ assert.equal(ran, false);
+ const [failure] = await errors.list({ sessionId: result.sessionId! });
+ assert.deepEqual(
+ { category: failure?.category, code: failure?.code, message: failure?.message },
+ { category: "turn", code: "detection_failed", message: "401 invalid x-api-key" },
+ );
+ const silent = (await auditLog.events()).find((e) => e.action === "turn.silent");
+ assert.equal(silent?.detail, "detection failed: 401 invalid x-api-key");
+});