A desktop coding agent with free access to frontier models.
TypeScript
fix: request GPT-6 reasoning summaries (#41)
Slendywyd83e0566785e
diff --git a/apps/server/src/omnirush-gateway-broker.test.ts b/apps/server/src/omnirush-gateway-broker.test.tsindex 16420a20..8cb5e23b 100644--- a/apps/server/src/omnirush-gateway-broker.test.ts+++ b/apps/server/src/omnirush-gateway-broker.test.ts@@ -153,11 +153,42 @@ describe("OmniRush gateway broker", () => {expect(response.status).toBe(200);expect(calls).toHaveLength(1);- expect(calls[0]?.body).toEqual({ model: "gpt-6-astra", input: "test", reasoning: { effort: "max" } });+ expect(calls[0]?.body).toEqual({ model: "gpt-6-astra", input: "test", reasoning: { effort: "max", summary: "auto" } });expect(calls[0]?.headers.has("x-omnirush-reasoning-effort")).toBe(false);expect(calls[0]?.headers.get("authorization")).toBe("Bearer access-token");});+ test("requests GPT-6 summaries without an effort header and preserves explicit options", async () => {+ const calls: UpstreamCall[] = [];+ const broker = capturingBroker(calls);+ const bodies = [+ { model: "gpt-6-astra", input: "default" },+ { model: "gpt-6-sol", input: "selected", reasoning: { effort: "high" } },+ { model: "gpt-6-astra", input: "legacy", reasoning_effort: "xhigh" },+ { model: "gpt-6-astra", input: "explicit", reasoning: { effort: "low", summary: "detailed" } },+ { model: "gpt-6-sol", input: "disabled", reasoning: { summary: null } },+ { model: "gpt-5.6-sol", input: "sol" },+ { model: "meta-muse", input: "other" },+ ];+ for (const body of bodies) await broker.handle(gatewayRequest(body), "responses");+ expect(calls.map((call) => call.body)).toEqual([+ { ...bodies[0], reasoning: { summary: "auto" } },+ { ...bodies[1], reasoning: { effort: "high", summary: "auto" } },+ { ...bodies[2], reasoning: { summary: "auto" } },+ ...bodies.slice(3),+ ]);+ });++ test("leaves compaction and malformed request bodies without a summary default", async () => {+ const calls: UpstreamCall[] = [];+ const broker = capturingBroker(calls);+ const compact = { model: "gpt-6-astra", input: "compact", reasoning: { effort: "high" } };+ const malformed = { model: "gpt-6-astra", input: "invalid", reasoning: "invalid" };+ await broker.handle(gatewayRequest(compact, {}, "responses/compact"), "responses/compact");+ await broker.handle(gatewayRequest(malformed), "responses");+ expect(calls.map((call) => call.body)).toEqual([compact, malformed]);+ });+test("keeps an effort the engine already emitted and the legacy top-level field", async () => {const calls: UpstreamCall[] = [];const broker = capturingBroker(calls);@@ -171,8 +202,8 @@ describe("OmniRush gateway broker", () => {expect(calls.map((call) => call.body)).toEqual([{ model: "gpt-5.6-sol", input: "a", reasoning: { effort: "max", summary: "auto" } },{ model: "gpt-5.6-sol", input: "b", reasoning_effort: "high" },- { model: "gpt-6-astra", input: "c" },- { model: "gpt-6-astra", input: "d" },+ { model: "gpt-6-astra", input: "c", reasoning: { summary: "auto" } },+ { model: "gpt-6-astra", input: "d", reasoning: { summary: "auto" } },]);});});@@ -1112,6 +1143,17 @@ describe("OmniRush gateway broker: sub-agent model fallback", () => {})]);});+ test("adds the summary default when an unrelated model falls back to Astra", async () => {+ const { broker, calls } = fallbackBroker((body) => body.model === "meta-muse-spark"+ ? refuse(400, "model_unavailable")+ : Response.json({ output: [] }));+ await broker.handle(gatewayRequest({ model: "meta-muse-spark", input: "fallback" }, subagentHeaders), "responses");+ expect(calls.map((call) => call.body)).toEqual([+ { model: "meta-muse-spark", input: "fallback" },+ { model: "gpt-6-astra", input: "fallback", reasoning: { effort: "max", summary: "auto" } },+ ]);+ });+test("a busy picked model is tried once more, then moves; a second success stays on it", async () => {let busy = 2;const stays = fallbackBroker((body) => body.model === "gpt-6-sol" && busy-- > 1@@ -1124,11 +1166,11 @@ describe("OmniRush gateway broker: sub-agent model fallback", () => {const moves = fallbackBroker((body) => body.model === "gpt-6-sol"? new Response("upstream down", { status: 503 }): Response.json({ output: [] }));- expect((await moves.broker.handle(gatewayRequest({ model: "gpt-6-sol", input: "b", reasoning_effort: "high" }, subagentHeaders), "responses")).status).toBe(200);+ expect((await moves.broker.handle(gatewayRequest({ model: "gpt-6-sol", input: "b", reasoning_effort: "high", reasoning: { summary: "auto" } }, subagentHeaders), "responses")).status).toBe(200);expect(moves.calls.map((call) => call.body)).toEqual([- { model: "gpt-6-sol", input: "b", reasoning_effort: "high" },- { model: "gpt-6-sol", input: "b", reasoning_effort: "high" },- { model: "gpt-6-astra", input: "b", reasoning: { effort: "max" } },+ { model: "gpt-6-sol", input: "b", reasoning_effort: "high", reasoning: { summary: "auto" } },+ { model: "gpt-6-sol", input: "b", reasoning_effort: "high", reasoning: { summary: "auto" } },+ { model: "gpt-6-astra", input: "b", reasoning: { effort: "max", summary: "auto" } },]);expect(moves.events.map((event) => event.reason)).toEqual(["http_503"]);});@@ -1162,7 +1204,7 @@ describe("OmniRush gateway broker: sub-agent model fallback", () => {test("a picked model in its refusal cooldown goes straight to the main model", async () => {const { broker, calls, events } = fallbackBroker(() => Response.json({ output: [] }), (model) => model === "gpt-6-sol");expect((await broker.handle(gatewayRequest({ model: "gpt-6-sol", input: "a" }, subagentHeaders), "responses")).status).toBe(200);- expect(calls.map((call) => call.body)).toEqual([{ model: "gpt-6-astra", input: "a", reasoning: { effort: "max" } }]);+ expect(calls.map((call) => call.body)).toEqual([{ model: "gpt-6-astra", input: "a", reasoning: { effort: "max", summary: "auto" } }]);expect(events.map((event) => [event.requested, event.used, event.reason, event.ok])).toEqual([["gpt-6-sol", "gpt-6-astra", "refused_recently", true]]);// Without the fallback header (the main agent, or an untouched setting) nothing moves.expect((await broker.handle(gatewayRequest({ model: "gpt-6-sol", input: "b" }), "responses")).status).toBe(200);diff --git a/apps/server/src/omnirush-gateway-broker.ts b/apps/server/src/omnirush-gateway-broker.tsindex a1036d48..b72007d4 100644--- a/apps/server/src/omnirush-gateway-broker.ts+++ b/apps/server/src/omnirush-gateway-broker.ts@@ -544,6 +544,26 @@ function withReasoningEffort(body: ArrayBuffer, effort: string): ArrayBuffer | sreturn JSON.stringify({ ...parsed, reasoning: { ...reasoning, effort } });}+/**+ * OpenCode currently opts into OpenAI summaries only for GPT-5 model IDs.+ * Ask for the summaries the transcript displays for our GPT-6 models too,+ * preserving any explicit summary option. Apply this at the final forwarding+ * boundary so sub-agent fallbacks use the destination model's options.+ */+function withReasoningSummary(body: ArrayBuffer | string): ArrayBuffer | string {+ let parsed: unknown;+ try {+ parsed = JSON.parse(typeof body === "string" ? body : new TextDecoder().decode(body));+ } catch {+ return body;+ }+ if (!isRecord(parsed) || (parsed.model !== "gpt-6-astra" && parsed.model !== "gpt-6-sol")) return body;+ if (parsed.reasoning != null && !isRecord(parsed.reasoning)) return body;+ const reasoning = isRecord(parsed.reasoning) ? parsed.reasoning : {};+ if ("summary" in reasoning) return body;+ return JSON.stringify({ ...parsed, reasoning: { ...reasoning, summary: "auto" } });+}+/*** Gateway refusals of a sub-agent's picked model that move the request to* the main model at once: the model is not served to this account (or at@@ -1028,6 +1048,7 @@ export class OmniRushGatewayBroker {body: ArrayBuffer | string | undefined,accessToken?: string,): Promise<Response> {+ const forwardedBody = path === "responses" && body !== undefined ? withReasoningSummary(body) : body;let failure: ReturnType<typeof fetchFailure> | null = null;for (let attempt = 0; attempt <= UNREACHABLE_RETRY_DELAYS_MS.length; attempt += 1) {if (attempt > 0) {@@ -1036,7 +1057,7 @@ export class OmniRushGatewayBroker {}if (!this.state) break;try {- return await this.forward(request, path, body, attempt === 0 ? accessToken : this.state.accessToken);+ return await this.forward(request, path, forwardedBody, attempt === 0 ? accessToken : this.state.accessToken);} catch (error) {if (request.signal.aborted) throw error;failure = fetchFailure(error);diff --git a/evals/specs/model-reasoning-summary.test.ts b/evals/specs/model-reasoning-summary.test.tsnew file mode 100644index 00000000..a900dde8--- /dev/null+++ b/evals/specs/model-reasoning-summary.test.ts@@ -0,0 +1,70 @@+import { expect } from "vitest";+import { spec } from "@omnirush/testkit";+import { modelReasoningSummary, REPLY, SUMMARY } from "../worlds/model-reasoning-summary.ts";++const test = spec.world(modelReasoningSummary, { needs: { commands: ["bun"] }, timeout: 240_000 });+function record(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++test("Astra and Sol deliver requested reasoning summaries through the desktop broker into persisted engine messages", async ({ world, step, evidence }) => {+ for (const modelID of ["gpt-6-astra", "gpt-6-sol", "gpt-5.6-sol"]) {+ await step(`reasoning remains available after switching to ${modelID}`, async () => {+ const session = await world.engine("POST", "/session", { title: "Summary verification" });+ if (!record(session) || typeof session.id !== "string") throw new Error("Missing session id");+ const before = world.requests.length;+ const result = await world.engine("POST", `/session/${session.id}/message`, {+ model: { providerID: "omnirush", modelID }, variant: "high",+ parts: [{ type: "text", text: "Compute 17 plus 25." }],+ });+ expect(record(result) && record(result.info) && result.info.error).toBeUndefined();+ if (!record(result) || !Array.isArray(result.parts)) throw new Error("Missing reply parts");+ expect(result.parts.filter(record).filter((part) => part.type === "reasoning")).toEqual([+ expect.objectContaining({ text: SUMMARY }),+ ]);+ expect(result.parts.filter(record).filter((part) => part.type === "text")).toEqual([+ expect.objectContaining({ text: REPLY }),+ ]);+ const requests = world.requests.slice(before);+ expect(requests).toHaveLength(1);+ expect(requests[0]).toMatchObject({ path: "/v1/responses", body: { model: modelID, reasoning: { effort: "high", summary: "auto" } },+ headers: { authorization: "Bearer summary-witness-access" } });+ expect(requests[0]?.headers["x-omnirush-reasoning-effort"]).toBeUndefined();+ const persisted = await world.engine("GET", `/session/${session.id}/message`);+ if (!Array.isArray(persisted)) throw new Error("Missing persisted messages");+ const assistant = persisted.filter(record).find((message) => record(message.info) && message.info.role === "assistant");+ expect(assistant).toMatchObject({ parts: expect.arrayContaining([expect.objectContaining({ type: "reasoning", text: SUMMARY })]) });+ });+ }++ await step("explicit settings, other models and compaction retain their request options", async () => {+ const bodies = [+ { model: "gpt-6-astra", input: "defaults" },+ { model: "gpt-6-sol", input: "explicit", reasoning: { effort: "low", summary: "detailed" } },+ { model: "gpt-6-astra", input: "disabled", reasoning: { summary: null } },+ { model: "gpt-5.6-sol", input: "no-default" },+ { model: "other-model", input: "unrelated" },+ ];+ const before = world.requests.length;+ for (const body of bodies) expect((await world.gateway(body)).status).toBe(200);+ const compact = { model: "gpt-6-astra", input: "compact", reasoning: { effort: "high" } };+ expect((await world.gateway(compact, "responses/compact")).status).toBe(200);+ expect(world.requests.slice(before).map((request) => request.body)).toEqual([+ { ...bodies[0], reasoning: { summary: "auto" } }, ...bodies.slice(1), compact,+ ]);+ });++ await step("a subagent fallback requests the destination Astra summary", async () => {+ const before = world.requests.length;+ expect((await world.gateway({ model: "unavailable-model", input: "fallback" }, "responses", {+ "x-omnirush-subagent-fallback-model": "gpt-6-astra",+ "x-omnirush-subagent-fallback-effort": "max",+ })).status).toBe(200);+ expect(world.requests.slice(before).map((request) => request.body)).toEqual([+ { model: "unavailable-model", input: "fallback" },+ { model: "gpt-6-astra", input: "fallback", reasoning: { effort: "max", summary: "auto" } },+ ]);+ });+ evidence.recordAssertionEvidence("GPT-6 summary opt-in reaches the transcript data", "The real embedded desktop server and bundled OpenCode engine send exactly one high-effort summary-enabled request for Astra, GPT 6 Sol and GPT-5.6 Sol; each returned and persisted a reasoning part plus the answer. The provider witness emits summary events only when explicitly requested.", true);+ evidence.recordAssertionEvidence("Request settings and routing remain scoped", "Observable gateway requests preserve explicit detailed and null summary values, leave other model IDs and compact requests unchanged, strip the private effort header, and add the summary after a subagent switches from an unrelated model to Astra.", true);+});diff --git a/evals/worlds/model-reasoning-summary.ts b/evals/worlds/model-reasoning-summary.tsnew file mode 100644index 00000000..2f67504e--- /dev/null+++ b/evals/worlds/model-reasoning-summary.ts@@ -0,0 +1,118 @@+import { spawn } from "node:child_process";+import { mkdir, rm } from "node:fs/promises";+import { createServer } from "node:http";+import { join, resolve } from "node:path";+import { SkipError, type Seed } from "@omnirush/env";+import { close, engineBinary, isRecord, listen, readBody, sendJson, sendStream, stopChild } from "./omnirush-server-cli.ts";++export const SUMMARY = "I will add the two numbers.";+export const REPLY = "42.";+export type SummaryRequest = { path: string; body: Record<string, unknown>; headers: Record<string, string | string[] | undefined> };++export async function modelReasoningSummary(seed: Seed) {+ const binary = engineBinary();+ if (!binary) throw new SkipError("set OMNIRUSH_OPENCODE_BIN or install opencode");+ const root = seed.tmpPath("model-reasoning-summary");+ const workspace = join(root, "workspace");+ const home = join(root, "home");+ await mkdir(workspace, { recursive: true });+ await mkdir(home, { recursive: true });+ const requests: SummaryRequest[] = [];+ const witness = createServer((request, response) => {+ void (async () => {+ if (request.method !== "POST") return sendJson(response, 200, { object: "list", data: [] });+ const body: unknown = JSON.parse(await readBody(request));+ if (!isRecord(body)) return sendJson(response, 400, {});+ if (!request.url?.startsWith("/v1/responses")) return sendJson(response, 200, {});+ requests.push({ path: request.url, body, headers: request.headers });+ if (request.headers.authorization !== "Bearer summary-witness-access") return sendJson(response, 401, {});+ if (body.model === "unavailable-model") return sendJson(response, 400, { detail: "model_unavailable" });+ const id = `resp_summary_${requests.length}`;+ const reason = { id: `${id}_reason`, type: "reasoning", summary: [{ type: "summary_text", text: SUMMARY }] };+ const item = { id: `${id}_message`, type: "message", role: "assistant", status: "completed",+ content: [{ type: "output_text", text: REPLY, annotations: [] }] };+ const summarized = isRecord(body.reasoning) && typeof body.reasoning.summary === "string";+ const output = summarized ? [reason, item] : [item];+ const final = { id, created_at: 1, model: body.model, status: "completed", output,+ usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 } };+ if (!body.stream) return sendJson(response, 200, final);+ const chunks: unknown[] = [+ { type: "response.created", response: { ...final, status: "in_progress", output: [] }, sequence_number: 0 },+ ];+ if (summarized) chunks.push(+ { type: "response.output_item.added", output_index: 0, item: { ...reason, summary: [] } },+ { type: "response.reasoning_summary_part.added", item_id: reason.id, output_index: 0, summary_index: 0, part: { type: "summary_text", text: "" } },+ { type: "response.reasoning_summary_text.delta", item_id: reason.id, output_index: 0, summary_index: 0, delta: SUMMARY },+ { type: "response.reasoning_summary_text.done", item_id: reason.id, output_index: 0, summary_index: 0, text: SUMMARY },+ { type: "response.reasoning_summary_part.done", item_id: reason.id, output_index: 0, summary_index: 0, part: reason.summary[0] },+ { type: "response.output_item.done", output_index: 0, item: reason },+ );+ const outputIndex = summarized ? 1 : 0;+ chunks.push(+ { type: "response.output_item.added", output_index: outputIndex, item: { ...item, status: "in_progress", content: [] } },+ { type: "response.output_text.delta", item_id: item.id, output_index: outputIndex, content_index: 0, delta: REPLY },+ { type: "response.output_item.done", output_index: outputIndex, item },+ { type: "response.completed", response: final },+ );+ sendStream(response, chunks);+ })().catch((error: unknown) => { if (!response.headersSent) sendJson(response, 500, { error: String(error) }); else response.destroy(); });+ });+ const upstream = await listen(witness);+ const inherited = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("OPENCODE") && !key.startsWith("OMNIRUSH_")));+ const script = `+ import { startEmbeddedServer } from "./src/embedded.ts";+ const server = await startEmbeddedServer({+ host: "127.0.0.1", port: 0, token: "summary-client", hostToken: "summary-host",+ workspaces: [process.env.SUMMARY_WORKSPACE], approvalMode: "auto", logRequests: false,+ manageOpencode: true, opencodeBin: process.env.SUMMARY_BINARY,+ omnirushGatewayCredentials: { gatewayUrl: process.env.SUMMARY_UPSTREAM + "/v1",+ accessToken: "summary-witness-access", refreshToken: "summary-witness-refresh" },+ });+ console.log("SUMMARY_SERVER " + JSON.stringify({ base: server.url, workspaceId: server.config.workspaces[0].id, engineToken: server.config.omnirushEngineToken }));+ process.once("SIGTERM", async () => { await server.stop(); process.exit(0); });+ await new Promise(() => {});+ `;+ const child = spawn("bun", ["--conditions=development", "--eval", script], {+ cwd: resolve(import.meta.dirname, "../../apps/server"), stdio: ["ignore", "pipe", "pipe"],+ env: { ...inherited, HOME: home, XDG_CONFIG_HOME: join(home, ".config"), XDG_DATA_HOME: join(home, ".local/share"),+ XDG_CACHE_HOME: join(home, ".cache"), XDG_STATE_HOME: join(home, ".local/state"),+ SUMMARY_WORKSPACE: workspace, SUMMARY_BINARY: binary, SUMMARY_UPSTREAM: upstream },+ });+ const dispose = async () => { await stopChild(child); await close(witness); await rm(root, { recursive: true, force: true }); };+ try {+ const connection = await new Promise<{ base: string; workspaceId: string; engineToken: string }>((ready, reject) => {+ let output = "";+ const timer = setTimeout(() => reject(new Error("Embedded summary server did not start: " + output.slice(-3000))), 90_000);+ child.on("error", (error) => { clearTimeout(timer); reject(error); });+ child.on("exit", (code) => { clearTimeout(timer); reject(new Error(`Summary server exited ${code}: ${output.slice(-3000)}`)); });+ const receive = (chunk: Buffer) => {+ output += chunk.toString();+ const match = output.match(/SUMMARY_SERVER (.+)\n/);+ if (!match) return;+ const value: unknown = JSON.parse(match[1]);+ if (!isRecord(value) || typeof value.base !== "string" || typeof value.workspaceId !== "string" || typeof value.engineToken !== "string") return reject(new Error("Invalid summary server connection"));+ clearTimeout(timer);+ ready({ base: value.base, workspaceId: value.workspaceId, engineToken: value.engineToken });+ };+ child.stdout?.on("data", receive);+ child.stderr?.on("data", receive);+ });+ const engine = async (method: string, path: string, body?: unknown): Promise<unknown> => {+ const response = await fetch(`${connection.base}/workspace/${encodeURIComponent(connection.workspaceId)}/opencode${path}`, {+ method, headers: { authorization: "Bearer summary-client", "content-type": "application/json" },+ body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(90_000),+ });+ const value: unknown = await response.json();+ if (!response.ok) throw new Error(`Engine ${path}: ${response.status}: ${JSON.stringify(value)}`);+ return value;+ };+ return {+ engine, requests,+ gateway: (body: Record<string, unknown>, path = "responses", headers: Record<string, string> = {}) =>+ fetch(`${connection.base}/omnirush-gateway/v1/${path}`, { method: "POST",+ headers: { authorization: `Bearer ${connection.engineToken}`, "content-type": "application/json", ...headers },+ body: JSON.stringify(body), signal: AbortSignal.timeout(15_000) }),+ [Symbol.asyncDispose]: dispose,+ };+ } catch (error) { await dispose(); throw error; }+}