Multiplayer agent harness for work.
Fix repeated provider history in stored prompt envelopes (#1380)
Regan Bell71c341555d17
diff --git a/src/harness/harness.ts b/src/harness/harness.tsindex 894d3dd1a..f99f05b2e 100644--- a/src/harness/harness.ts+++ b/src/harness/harness.ts@@ -30,9 +30,27 @@ interface HarnessImage {artifactId?: string;}-export function envelopeWithoutMessages(payload: unknown): unknown {+export function promptEnvelopeWithoutHistory(payload: unknown): unknown {if (!payload || typeof payload !== "object" || Array.isArray(payload)) return payload;- return Object.fromEntries(Object.entries(payload as Record<string, unknown>).filter(([k]) => k !== "messages"));+ const envelope: Record<string, unknown> = {};+ for (const [key, value] of Object.entries(payload)) {+ if (key === "contents") continue;+ if (key === "messages" || key === "input") {+ if (Array.isArray(value)) {+ const instructions = value.filter(+ (item: unknown) =>+ item !== null &&+ typeof item === "object" &&+ "role" in item &&+ (item.role === "system" || item.role === "developer"),+ );+ if (instructions.length) envelope[key] = instructions;+ }+ } else {+ envelope[key] = key === "context" ? promptEnvelopeWithoutHistory(value) : value;+ }+ }+ return envelope;}export interface HarnessLlmRequestRecord {diff --git a/src/harness/opencode-harness.ts b/src/harness/opencode-harness.tsindex 727115cd3..5dc69451c 100644--- a/src/harness/opencode-harness.ts+++ b/src/harness/opencode-harness.ts@@ -25,7 +25,7 @@ import { sleep } from "../util/async.ts";import { NonRetryableTurnError } from "../core/turn-error.ts";import {defineHarness,- envelopeWithoutMessages,+ promptEnvelopeWithoutHistory,type Harness,type HarnessTurnInput,type HarnessTurnResult,@@ -1003,7 +1003,7 @@ export function createOpenCodeHarness(opts: OpenCodeHarnessOptions = {}): HarnesturnSeq: state.userSeq,step: capture.step,model: capture.model,- promptEnvelope: envelopeWithoutMessages(capture.request),+ promptEnvelope: promptEnvelopeWithoutHistory(capture.request),truncated: false,transport: info?.providerID && info.modelID ? { modelId: `${info.providerID}/${info.modelID}` } : null,ttftMs: null,diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.tsindex 3d30a764f..fb5373adb 100644--- a/src/harness/pi-harness.ts+++ b/src/harness/pi-harness.ts@@ -68,7 +68,7 @@ import { customModelsJson, customProvidersVersion } from "../model/custom-providimport { modelGatewayRequest, type ModelGatewayTransportConfig } from "../model/provider-endpoints.ts";import {defineHarness,- envelopeWithoutMessages,+ promptEnvelopeWithoutHistory,type Harness,type HarnessCompactInput,type HarnessDetectInput,@@ -854,7 +854,7 @@ export function sanitizeLlmPayload(const withTransport = (r: { envelope: unknown; truncated: boolean }) => (transport ? { ...r, transport } : r);let redacted: unknown;try {- redacted = redactImageBytes(envelopeWithoutMessages(payload));+ redacted = redactImageBytes(promptEnvelopeWithoutHistory(payload));} catch {return withTransport({ envelope: { note: "payload not capturable" }, truncated: true });}diff --git a/test/opencode-harness.test.ts b/test/opencode-harness.test.tsindex 91a04ded1..eb54c7bf5 100644--- a/test/opencode-harness.test.ts+++ b/test/opencode-harness.test.ts@@ -252,6 +252,10 @@ test("OpenCode records requests without usage attribution when captures and assiconst llmRows: HarnessLlmRequestRecord[] = [];const result = await harness.turns.runTurn(turnInput([], llmRows));assert.equal(result.reply, "hello from fake");+ assert.deepEqual(+ llmRows.map((row) => row.promptEnvelope),+ [{ system: "s" }, { system: "s" }],+ );assert.deepEqual(llmRows.map((row) => ({ step: row.step, usage: row.usage, durationMs: row.durationMs })),[diff --git a/test/pi-harness-tape-sync.test.ts b/test/pi-harness-tape-sync.test.tsindex 486a7a3ac..2be768851 100644--- a/test/pi-harness-tape-sync.test.ts+++ b/test/pi-harness-tape-sync.test.ts@@ -2,8 +2,8 @@ import { readFile } from "node:fs/promises";import { test } from "node:test";import assert from "node:assert/strict";import { createPiHarness } from "../src/harness/pi-harness.ts";-import type { HarnessTurnInput } from "../src/harness/harness.ts";-import type { NewEntry, NewTapeRecord } from "../src/sessions/session-store.ts";+import type { HarnessLlmRequestRecord, HarnessTurnInput } from "../src/harness/harness.ts";+import { promptEnvelopeBody, type NewEntry, type NewTapeRecord } from "../src/sessions/session-store.ts";import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";import { createMemoryRunSignalStore } from "../src/runs/run-signal-store.ts";import type { ScopeId, SessionEntry } from "../src/types.ts";@@ -328,10 +328,10 @@ test("native document bytes reach the provider on every step without entering th}});-test("Pi request captures exclude document fallback text on Responses routes", async () => {+test("Pi Responses captures exclude history and document text without changing requests or tape", async () => {const harness = createPiHarness({ openaiApiKey: "sk-test", modelId: "gpt-5.6-sol" });const sink: Sink = { entries: [], tape: [] };- const captures: unknown[] = [];+ const captures: HarnessLlmRequestRecord[] = [];const realFetch = globalThis.fetch;const requests: unknown[] = [];globalThis.fetch = (async (_url: unknown, init?: RequestInit) => {@@ -366,6 +366,17 @@ test("Pi request captures exclude document fallback text on Responses routes", a},}),);+ assert.ok(requests.length > 1);+ assert.match(JSON.stringify(requests[0]), /do the thing/);+ assert.match(JSON.stringify(requests.at(-1)), /do the thing/);+ assert.match(JSON.stringify(requests.at(-1)), /The turn ended with an empty message/);+ assert.match(JSON.stringify(sink.tape), /do the thing/);+ const firstEnvelope = promptEnvelopeBody(captures[0]!.promptEnvelope)!;+ assert.match(firstEnvelope.body, /BASE/);+ for (const capture of captures) {+ assert.equal(promptEnvelopeBody(capture.promptEnvelope)!.hash, firstEnvelope.hash);+ assert.doesNotMatch(JSON.stringify(capture.promptEnvelope), /do the thing|The turn ended with an empty message/);+ }assert.ok(JSON.stringify(requests).includes("RTF-QUARTZ-731"));assert.ok(captures.length > 0);assert.ok(!JSON.stringify(captures).includes("RTF-QUARTZ-731"));diff --git a/test/prompt-envelope.test.ts b/test/prompt-envelope.test.tsnew file mode 100644index 000000000..5710cf237--- /dev/null+++ b/test/prompt-envelope.test.ts@@ -0,0 +1,118 @@+import assert from "node:assert/strict";+import { test } from "node:test";+import type { Context } from "@earendil-works/pi-ai";+import { promptEnvelopeWithoutHistory } from "../src/harness/harness.ts";+import { sanitizeLlmPayload } from "../src/harness/pi-harness.ts";+import { promptEnvelopeBody } from "../src/sessions/session-store.ts";++const cases: Array<{+ name: string;+ payload: (history: string[], instruction: string, tool: string) => unknown;+}> = [+ {+ name: "Anthropic and Bedrock messages",+ payload: (history, instruction, tool) => ({+ model: "test-model",+ system: [{ text: instruction }],+ tools: [{ name: tool }],+ messages: history.map((text) => ({ role: "user", content: [{ text }] })),+ }),+ },+ {+ name: "Chat Completions and Mistral messages",+ payload: (history, instruction, tool) => ({+ model: "test-model",+ temperature: 0.5,+ tools: [{ type: "function", function: { name: tool } }],+ messages: [+ { role: "system", content: instruction },+ ...history.map((content) => ({ role: "user" as const, content })),+ { role: "developer", content: "developer instruction" },+ { role: "assistant", content: "answer" },+ { role: "tool", tool_call_id: "call_1", content: "result" },+ ],+ }),+ },+ {+ name: "OpenAI and Azure Responses input",+ payload: (history, instruction, tool) => ({+ model: "test-model",+ tools: [{ type: "function", name: tool, parameters: {}, strict: false }],+ input: [+ { role: "system", content: instruction },+ ...history.map((text) => ({ role: "user" as const, content: [{ type: "input_text" as const, text }] })),+ { type: "message", role: "developer", content: [{ type: "input_text", text: "developer instruction" }] },+ { type: "function_call", call_id: "call_1", name: tool, arguments: "{}" },+ { type: "function_call_output", call_id: "call_1", output: "result" },+ { type: "reasoning", id: "rs_1", summary: [], encrypted_content: "reasoning" },+ ],+ }),+ },+ {+ name: "Codex Responses instructions",+ payload: (history, instruction, tool) => ({+ model: "test-model",+ instructions: instruction,+ tools: [{ type: "function", name: tool, parameters: {}, strict: false }],+ input: history.map((content) => ({ role: "user", content })),+ }),+ },+ {+ name: "Responses string input",+ payload: (history, instruction, tool) => ({+ model: "test-model",+ instructions: instruction,+ tools: [{ type: "function", name: tool, parameters: {}, strict: false }],+ input: history.join("\n"),+ }),+ },+ {+ name: "Gemini and Vertex contents",+ payload: (history, instruction, tool) => ({+ model: "test-model",+ config: { systemInstruction: instruction, tools: [{ functionDeclarations: [{ name: tool }] }] },+ contents: history.map((text) => ({ role: "user", parts: [{ text }] })),+ }),+ },+ {+ name: "Pi messages context",+ payload: (history, instruction, tool) => ({+ model: "test-model",+ options: { temperature: 0.5 },+ context: {+ systemPrompt: instruction,+ tools: [{ name: tool, description: "test tool", parameters: { type: "object", properties: {} } }],+ messages: history.map((content) => ({ role: "user", content, timestamp: 1 })),+ } satisfies Context,+ }),+ },+];++for (const { name, payload } of cases) {+ test(`${name}: capture hashes ignore history but retain instructions and tools without mutation`, () => {+ const first = payload(["hello"], "be helpful", "lookup");+ const grown = payload(["hello", "another user message", "tool result"], "be helpful", "lookup");+ const original = structuredClone(grown);+ for (const extract of [promptEnvelopeWithoutHistory, (value: unknown) => sanitizeLlmPayload(value).envelope]) {+ const envelope = extract(grown);+ assert.deepEqual(grown, original);+ const body = promptEnvelopeBody(envelope)!;+ assert.equal(body.hash, promptEnvelopeBody(extract(first))!.hash);+ assert.notEqual(body.hash, promptEnvelopeBody(extract(payload([], "new instruction", "lookup")))!.hash);+ assert.notEqual(body.hash, promptEnvelopeBody(extract(payload([], "be helpful", "other_tool")))!.hash);+ assert.match(body.body, /be helpful/);+ assert.match(body.body, /lookup/);+ assert.doesNotMatch(body.body, /hello|another user message|tool result|reasoning|call_1/);+ if (name.includes("Chat") || name.includes("Azure")) assert.match(body.body, /developer instruction/);+ }+ assert.equal(sanitizeLlmPayload(grown).truncated, false);+ });+}++test("empty histories and malformed payloads do not add envelope fields or throw", () => {+ for (const value of [null, undefined, "text", 12, []]) assert.equal(promptEnvelopeWithoutHistory(value), value);+ for (const value of [null, "history", [], [{ role: "user", content: "hello" }]]) {+ assert.deepEqual(promptEnvelopeWithoutHistory({ messages: value, input: value, contents: value }), {});+ }+ assert.deepEqual(promptEnvelopeWithoutHistory({ input: [null, 1, {}, { role: "user" }] }), {});+});