Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Fix repeated provider history in stored prompt envelopes (#1380)

Regan Bell71c341555d17
diff --git a/src/harness/harness.ts b/src/harness/harness.ts
index 894d3dd1a..f99f05b2e 100644
--- a/src/harness/harness.ts
+++ b/src/harness/harness.ts
@@ -30,9 +30,27 @@ interface HarnessImage {
artifactId?: string;
}
-export function envelopeWithoutMessages(payload: unknown): unknown {
+export function promptEnvelopeWithoutHistory(payload: unknown): unknown {
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return payload;
- return Object.fromEntries(Object.entries(payload as Record<string, unknown>).filter(([k]) => k !== "messages"));
+ const envelope: Record<string, unknown> = {};
+ for (const [key, value] of Object.entries(payload)) {
+ if (key === "contents") continue;
+ if (key === "messages" || key === "input") {
+ if (Array.isArray(value)) {
+ const instructions = value.filter(
+ (item: unknown) =>
+ item !== null &&
+ typeof item === "object" &&
+ "role" in item &&
+ (item.role === "system" || item.role === "developer"),
+ );
+ if (instructions.length) envelope[key] = instructions;
+ }
+ } else {
+ envelope[key] = key === "context" ? promptEnvelopeWithoutHistory(value) : value;
+ }
+ }
+ return envelope;
}
export interface HarnessLlmRequestRecord {
diff --git a/src/harness/opencode-harness.ts b/src/harness/opencode-harness.ts
index 727115cd3..5dc69451c 100644
--- a/src/harness/opencode-harness.ts
+++ b/src/harness/opencode-harness.ts
@@ -25,7 +25,7 @@ import { sleep } from "../util/async.ts";
import { NonRetryableTurnError } from "../core/turn-error.ts";
import {
defineHarness,
- envelopeWithoutMessages,
+ promptEnvelopeWithoutHistory,
type Harness,
type HarnessTurnInput,
type HarnessTurnResult,
@@ -1003,7 +1003,7 @@ export function createOpenCodeHarness(opts: OpenCodeHarnessOptions = {}): Harnes
turnSeq: state.userSeq,
step: capture.step,
model: capture.model,
- promptEnvelope: envelopeWithoutMessages(capture.request),
+ promptEnvelope: promptEnvelopeWithoutHistory(capture.request),
truncated: false,
transport: info?.providerID && info.modelID ? { modelId: `${info.providerID}/${info.modelID}` } : null,
ttftMs: null,
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index 3d30a764f..fb5373adb 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -68,7 +68,7 @@ import { customModelsJson, customProvidersVersion } from "../model/custom-provid
import { modelGatewayRequest, type ModelGatewayTransportConfig } from "../model/provider-endpoints.ts";
import {
defineHarness,
- envelopeWithoutMessages,
+ promptEnvelopeWithoutHistory,
type Harness,
type HarnessCompactInput,
type HarnessDetectInput,
@@ -854,7 +854,7 @@ export function sanitizeLlmPayload(
const withTransport = (r: { envelope: unknown; truncated: boolean }) => (transport ? { ...r, transport } : r);
let redacted: unknown;
try {
- redacted = redactImageBytes(envelopeWithoutMessages(payload));
+ redacted = redactImageBytes(promptEnvelopeWithoutHistory(payload));
} catch {
return withTransport({ envelope: { note: "payload not capturable" }, truncated: true });
}
diff --git a/test/opencode-harness.test.ts b/test/opencode-harness.test.ts
index 91a04ded1..eb54c7bf5 100644
--- a/test/opencode-harness.test.ts
+++ b/test/opencode-harness.test.ts
@@ -252,6 +252,10 @@ test("OpenCode records requests without usage attribution when captures and assi
const llmRows: HarnessLlmRequestRecord[] = [];
const result = await harness.turns.runTurn(turnInput([], llmRows));
assert.equal(result.reply, "hello from fake");
+ assert.deepEqual(
+ llmRows.map((row) => row.promptEnvelope),
+ [{ system: "s" }, { system: "s" }],
+ );
assert.deepEqual(
llmRows.map((row) => ({ step: row.step, usage: row.usage, durationMs: row.durationMs })),
[
diff --git a/test/pi-harness-tape-sync.test.ts b/test/pi-harness-tape-sync.test.ts
index 486a7a3ac..2be768851 100644
--- a/test/pi-harness-tape-sync.test.ts
+++ b/test/pi-harness-tape-sync.test.ts
@@ -2,8 +2,8 @@ import { readFile } from "node:fs/promises";
import { test } from "node:test";
import assert from "node:assert/strict";
import { createPiHarness } from "../src/harness/pi-harness.ts";
-import type { HarnessTurnInput } from "../src/harness/harness.ts";
-import type { NewEntry, NewTapeRecord } from "../src/sessions/session-store.ts";
+import type { HarnessLlmRequestRecord, HarnessTurnInput } from "../src/harness/harness.ts";
+import { promptEnvelopeBody, type NewEntry, type NewTapeRecord } from "../src/sessions/session-store.ts";
import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";
import { createMemoryRunSignalStore } from "../src/runs/run-signal-store.ts";
import type { ScopeId, SessionEntry } from "../src/types.ts";
@@ -328,10 +328,10 @@ test("native document bytes reach the provider on every step without entering th
}
});
-test("Pi request captures exclude document fallback text on Responses routes", async () => {
+test("Pi Responses captures exclude history and document text without changing requests or tape", async () => {
const harness = createPiHarness({ openaiApiKey: "sk-test", modelId: "gpt-5.6-sol" });
const sink: Sink = { entries: [], tape: [] };
- const captures: unknown[] = [];
+ const captures: HarnessLlmRequestRecord[] = [];
const realFetch = globalThis.fetch;
const requests: unknown[] = [];
globalThis.fetch = (async (_url: unknown, init?: RequestInit) => {
@@ -366,6 +366,17 @@ test("Pi request captures exclude document fallback text on Responses routes", a
},
}),
);
+ assert.ok(requests.length > 1);
+ assert.match(JSON.stringify(requests[0]), /do the thing/);
+ assert.match(JSON.stringify(requests.at(-1)), /do the thing/);
+ assert.match(JSON.stringify(requests.at(-1)), /The turn ended with an empty message/);
+ assert.match(JSON.stringify(sink.tape), /do the thing/);
+ const firstEnvelope = promptEnvelopeBody(captures[0]!.promptEnvelope)!;
+ assert.match(firstEnvelope.body, /BASE/);
+ for (const capture of captures) {
+ assert.equal(promptEnvelopeBody(capture.promptEnvelope)!.hash, firstEnvelope.hash);
+ assert.doesNotMatch(JSON.stringify(capture.promptEnvelope), /do the thing|The turn ended with an empty message/);
+ }
assert.ok(JSON.stringify(requests).includes("RTF-QUARTZ-731"));
assert.ok(captures.length > 0);
assert.ok(!JSON.stringify(captures).includes("RTF-QUARTZ-731"));
diff --git a/test/prompt-envelope.test.ts b/test/prompt-envelope.test.ts
new file mode 100644
index 000000000..5710cf237
--- /dev/null
+++ b/test/prompt-envelope.test.ts
@@ -0,0 +1,118 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import type { Context } from "@earendil-works/pi-ai";
+import { promptEnvelopeWithoutHistory } from "../src/harness/harness.ts";
+import { sanitizeLlmPayload } from "../src/harness/pi-harness.ts";
+import { promptEnvelopeBody } from "../src/sessions/session-store.ts";
+
+const cases: Array<{
+ name: string;
+ payload: (history: string[], instruction: string, tool: string) => unknown;
+}> = [
+ {
+ name: "Anthropic and Bedrock messages",
+ payload: (history, instruction, tool) => ({
+ model: "test-model",
+ system: [{ text: instruction }],
+ tools: [{ name: tool }],
+ messages: history.map((text) => ({ role: "user", content: [{ text }] })),
+ }),
+ },
+ {
+ name: "Chat Completions and Mistral messages",
+ payload: (history, instruction, tool) => ({
+ model: "test-model",
+ temperature: 0.5,
+ tools: [{ type: "function", function: { name: tool } }],
+ messages: [
+ { role: "system", content: instruction },
+ ...history.map((content) => ({ role: "user" as const, content })),
+ { role: "developer", content: "developer instruction" },
+ { role: "assistant", content: "answer" },
+ { role: "tool", tool_call_id: "call_1", content: "result" },
+ ],
+ }),
+ },
+ {
+ name: "OpenAI and Azure Responses input",
+ payload: (history, instruction, tool) => ({
+ model: "test-model",
+ tools: [{ type: "function", name: tool, parameters: {}, strict: false }],
+ input: [
+ { role: "system", content: instruction },
+ ...history.map((text) => ({ role: "user" as const, content: [{ type: "input_text" as const, text }] })),
+ { type: "message", role: "developer", content: [{ type: "input_text", text: "developer instruction" }] },
+ { type: "function_call", call_id: "call_1", name: tool, arguments: "{}" },
+ { type: "function_call_output", call_id: "call_1", output: "result" },
+ { type: "reasoning", id: "rs_1", summary: [], encrypted_content: "reasoning" },
+ ],
+ }),
+ },
+ {
+ name: "Codex Responses instructions",
+ payload: (history, instruction, tool) => ({
+ model: "test-model",
+ instructions: instruction,
+ tools: [{ type: "function", name: tool, parameters: {}, strict: false }],
+ input: history.map((content) => ({ role: "user", content })),
+ }),
+ },
+ {
+ name: "Responses string input",
+ payload: (history, instruction, tool) => ({
+ model: "test-model",
+ instructions: instruction,
+ tools: [{ type: "function", name: tool, parameters: {}, strict: false }],
+ input: history.join("\n"),
+ }),
+ },
+ {
+ name: "Gemini and Vertex contents",
+ payload: (history, instruction, tool) => ({
+ model: "test-model",
+ config: { systemInstruction: instruction, tools: [{ functionDeclarations: [{ name: tool }] }] },
+ contents: history.map((text) => ({ role: "user", parts: [{ text }] })),
+ }),
+ },
+ {
+ name: "Pi messages context",
+ payload: (history, instruction, tool) => ({
+ model: "test-model",
+ options: { temperature: 0.5 },
+ context: {
+ systemPrompt: instruction,
+ tools: [{ name: tool, description: "test tool", parameters: { type: "object", properties: {} } }],
+ messages: history.map((content) => ({ role: "user", content, timestamp: 1 })),
+ } satisfies Context,
+ }),
+ },
+];
+
+for (const { name, payload } of cases) {
+ test(`${name}: capture hashes ignore history but retain instructions and tools without mutation`, () => {
+ const first = payload(["hello"], "be helpful", "lookup");
+ const grown = payload(["hello", "another user message", "tool result"], "be helpful", "lookup");
+ const original = structuredClone(grown);
+ for (const extract of [promptEnvelopeWithoutHistory, (value: unknown) => sanitizeLlmPayload(value).envelope]) {
+ const envelope = extract(grown);
+ assert.deepEqual(grown, original);
+ const body = promptEnvelopeBody(envelope)!;
+ assert.equal(body.hash, promptEnvelopeBody(extract(first))!.hash);
+ assert.notEqual(body.hash, promptEnvelopeBody(extract(payload([], "new instruction", "lookup")))!.hash);
+ assert.notEqual(body.hash, promptEnvelopeBody(extract(payload([], "be helpful", "other_tool")))!.hash);
+ assert.match(body.body, /be helpful/);
+ assert.match(body.body, /lookup/);
+ assert.doesNotMatch(body.body, /hello|another user message|tool result|reasoning|call_1/);
+ if (name.includes("Chat") || name.includes("Azure")) assert.match(body.body, /developer instruction/);
+ }
+ assert.equal(sanitizeLlmPayload(grown).truncated, false);
+ });
+}
+
+test("empty histories and malformed payloads do not add envelope fields or throw", () => {
+ for (const value of [null, undefined, "text", 12, []]) assert.equal(promptEnvelopeWithoutHistory(value), value);
+ for (const value of [null, "history", [], [{ role: "user", content: "hello" }]]) {
+ assert.deepEqual(promptEnvelopeWithoutHistory({ messages: value, input: value, contents: value }), {});
+ }
+ assert.deepEqual(promptEnvelopeWithoutHistory({ input: [null, 1, {}, { role: "user" }] }), {});
+});