Multiplayer agent harness for work.
Record reply-detection failures instead of discarding them (#2475)
Regan Bellcd48d3fe1e8a
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.tsindex c76efcbc0..687b577a3 100644--- a/src/core/orchestrator.ts+++ b/src/core/orchestrator.ts@@ -92,6 +92,7 @@ import {} from "../auth/capability-token.ts";import type {GapWork,+ HarnessDetectResult,HarnessLlmRequestRecord,HarnessTurnInput,HarnessTurnResult,@@ -2348,21 +2349,35 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {) {const detectHistory = filterHistory((await historyWindow()).entries.slice(-DETECT_HISTORY_TAIL));const detectStart = Date.now();- const decision = await deps.harness.models.shouldRespond({- session,- message: input.text,- recentContext: input.detectContext ?? "",- ...(input.detectOpener ? { threadOpener: input.detectOpener } : {}),- systemPrompt: resolution.systemPrompt,- ...(input.gatewayContext?.reactionGuidance- ? { reactionGuidance: input.gatewayContext.reactionGuidance }- : {}),- history: detectHistory,- recordModelCall: (rec) => {- deps.modelGateway.recordCall({ at: Date.now(), scopeLabel: scopeId, ...rec });- void deps.budget?.record(actor.id, estimateCostUsd(rec.inputTokens));- },- });+ const decision = await deps.harness.models+ .shouldRespond({+ session,+ message: input.text,+ recentContext: input.detectContext ?? "",+ ...(input.detectOpener ? { threadOpener: input.detectOpener } : {}),+ systemPrompt: resolution.systemPrompt,+ ...(input.gatewayContext?.reactionGuidance+ ? { reactionGuidance: input.gatewayContext.reactionGuidance }+ : {}),+ history: detectHistory,+ recordModelCall: (rec) => {+ deps.modelGateway.recordCall({ at: Date.now(), scopeLabel: scopeId, ...rec });+ void deps.budget?.record(actor.id, estimateCostUsd(rec.inputTokens));+ },+ })+ .catch((e: unknown): HarnessDetectResult => {+ deps.errors?.record(+ {+ category: "turn",+ code: "detection_failed",+ message: errMessage(e),+ scopeLabel: scopeId,+ sessionId: session.id,+ },+ e,+ );+ return { respond: false, reason: `detection failed: ${errMessage(e)}` };+ });detectMs = Date.now() - detectStart;if (!decision.respond) {const reactions = decision.reactions?.length ? decision.reactions : undefined;diff --git a/src/harness/claude-harness.ts b/src/harness/claude-harness.tsindex df6485d88..d6677c276 100644--- a/src/harness/claude-harness.ts+++ b/src/harness/claude-harness.ts@@ -858,19 +858,14 @@ export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness {},resetSession: () => {},async shouldRespond(detect) {- try {- const out = await single(- buildDetectionPrompt(detect.reactionGuidance),- renderDetectPrompt(detect),- undefined,- { recordModelCall: detect.recordModelCall },- judgeModelId,- );- return parseDetectVerdict((out ?? "").trim(), Boolean(detect.reactionGuidance?.trim()));- } catch (error) {- swallow("claude: detect", error);- return { respond: false };- }+ const out = await single(+ buildDetectionPrompt(detect.reactionGuidance),+ renderDetectPrompt(detect),+ undefined,+ { recordModelCall: detect.recordModelCall },+ judgeModelId,+ );+ return parseDetectVerdict((out ?? "").trim(), Boolean(detect.reactionGuidance?.trim()));},async compactHistory(input) {const model = getRequiredModel(resolveModelId());diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.tsindex 4683a59ac..bd25ffc62 100644--- a/src/harness/pi-harness.ts+++ b/src/harness/pi-harness.ts@@ -2406,25 +2406,21 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {},async shouldRespond(detect: HarnessDetectInput): Promise<HarnessDetectResult> {- try {- const modelId = detectModelId();- const model = requiredModel(modelId);- const providerKeys = await resolveProviderKeys();- if (!keyForModel(providerKeys, model)) return { respond: true };- const detectSystemPrompt = buildDetectionPrompt(detect.reactionGuidance);- const prompt = renderDetectPrompt(detect);- detect.recordModelCall({- model: modelId,- inputTokens: countTokens(detectSystemPrompt) + countTokens(prompt),- entryCount: detect.history.length,- });- const out = (- (await oneShot("pi-detect", model, providerKeys, detectSystemPrompt, prompt, { modelGateway })) ?? ""- ).trim();- return parseDetectVerdict(out, Boolean(detect.reactionGuidance?.trim()));- } catch {- return { respond: false };- }+ const modelId = detectModelId();+ const model = requiredModel(modelId);+ const providerKeys = await resolveProviderKeys();+ if (!keyForModel(providerKeys, model)) return { respond: true };+ const detectSystemPrompt = buildDetectionPrompt(detect.reactionGuidance);+ const prompt = renderDetectPrompt(detect);+ detect.recordModelCall({+ model: modelId,+ inputTokens: countTokens(detectSystemPrompt) + countTokens(prompt),+ entryCount: detect.history.length,+ });+ const out = (+ (await oneShot("pi-detect", model, providerKeys, detectSystemPrompt, prompt, { modelGateway })) ?? ""+ ).trim();+ return parseDetectVerdict(out, Boolean(detect.reactionGuidance?.trim()));},async compactHistory(input: HarnessCompactInput): Promise<string> {diff --git a/test/reply-detection-failure.test.ts b/test/reply-detection-failure.test.tsnew file mode 100644index 000000000..a7cb17978--- /dev/null+++ b/test/reply-detection-failure.test.ts@@ -0,0 +1,96 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { createOrchestrator } from "../src/core/orchestrator.ts";+import { createIdentityService } from "../src/identity/identity-service.ts";+import { createMemoryConfigStore } from "../src/resolution/config-store.ts";+import { createAclStore } from "../src/acl/acl-store.ts";+import { createResolutionService } from "../src/resolution/resolution-service.ts";+import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";+import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";+import { createMemoryService } from "../src/memory/memory-service.ts";+import { createModelGateway } from "../src/model/model-gateway.ts";+import { createAuditLog } from "../src/audit/audit-log.ts";+import { createErrorLog } from "../src/admin/error-log.ts";+import { createRateLimiter } from "../src/ratelimit/rate-limiter.ts";+import { createMockHarness } from "../src/harness/mock-harness.ts";+import { createDeployStore } from "../src/deploy/deploy-store.ts";+import { createDockerDeployProvider } from "../src/deploy/docker-deploy-provider.ts";+import { createDeployService } from "../src/deploy/deploy-service.ts";+import { createMemoryFileArtifactStore } from "../src/files/file-artifact-store.ts";+import { createMemoryDurableByteStore } from "../src/files/durable-byte-store.ts";+import type { Harness } from "../src/harness/harness.ts";+import type { Sandbox } from "../src/sandbox/sandbox.ts";+import type { Conversation, Principal } from "../src/types.ts";++const actor: Principal = { id: "U1", type: "internal" };++test("an ambient message whose reply detection fails stays silent and leaves the failure in the error and audit logs", async () => {+ const base = createMockHarness();+ let ran = false;+ const harness: Harness = {+ ...base,+ turns: {+ ...base.turns,+ runTurn: async () => {+ ran = true;+ return { reply: "spurious" };+ },+ },+ models: {+ ...base.models,+ shouldRespond: async () => {+ throw new Error("401 invalid x-api-key");+ },+ },+ };+ const config = createMemoryConfigStore("default-org");+ const acl = createAclStore();+ const auditLog = createAuditLog();+ const errors = createErrorLog();+ const workspace = createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "rdf-")));+ const orch = createOrchestrator({+ identity: createIdentityService(),+ resolution: createResolutionService("default-org", config, acl),+ sessions: createMemorySessionStore(),+ workspace,+ files: createMemoryFileArtifactStore(createMemoryDurableByteStore()),+ sandbox: {+ profile: { backend: "fake", writablePersistence: "snapshot_to_workspace", processSessions: false },+ } as Sandbox,+ modelGateway: createModelGateway(),+ auditLog,+ errors,+ rateLimiter: createRateLimiter({ maxPerWindow: 1000, windowMs: 60_000 }),+ harness,+ memory: createMemoryService(workspace),+ deploy: createDeployService({+ deployStore: createDeployStore(),+ provider: createDockerDeployProvider(),+ deployDir: join(tmpdir(), "rdf-deploy"),+ auditLog,+ acl,+ }),+ acl,+ });++ const result = await orch.handleTurn({+ surface: "slack",+ actor,+ conversation: { kind: "channel", threadRef: "ch:C1:200.1", channelRef: "C1", audience: [actor] } as Conversation,+ origin: { kind: "ambient" },+ text: "anyone know why the deploy is stuck?",+ });++ assert.equal(result.status, "silent");+ assert.equal(ran, false);+ const [failure] = await errors.list({ sessionId: result.sessionId! });+ assert.deepEqual(+ { category: failure?.category, code: failure?.code, message: failure?.message },+ { category: "turn", code: "detection_failed", message: "401 invalid x-api-key" },+ );+ const silent = (await auditLog.events()).find((e) => e.action === "turn.silent");+ assert.equal(silent?.detail, "detection failed: 401 invalid x-api-key");+});