Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

fix: make Auto reasoning provider-native and capability-gated (#1610)

Regan Bell74cb013d3d49
diff --git a/plugins/admin/test/settings-lit.test.ts b/plugins/admin/test/settings-lit.test.ts
index 631eaccdf..b7a039d7b 100644
--- a/plugins/admin/test/settings-lit.test.ts
+++ b/plugins/admin/test/settings-lit.test.ts
@@ -161,3 +161,45 @@ test("credential save acknowledges its submitted snapshot and retains newer edit
dom.window.close();
}
});
+
+test("runtime reasoning choices follow the selected model and preserve legacy defaults", () => {
+ const dom = setup();
+ try {
+ const data = {
+ ...models,
+ modelsByHarness: {
+ pi: [
+ { id: "a", effortLevels: ["auto", "adaptive", "default", "high"] },
+ { id: "b", effortLevels: ["auto", "high"] },
+ ],
+ },
+ thinkingLevelsByHarness: { pi: ["auto", "adaptive", "default", "high"] },
+ runtime: { harnessId: "pi", modelId: "a", effortLevel: "auto" },
+ };
+ dom.window.eval("settingsUI.load(" + JSON.stringify(data) + ',"org:test","runtime")');
+ const select = () => dom.window.document.getElementById("base-effort") as HTMLSelectElement;
+ const choices = () => [...select().options].map((option) => [option.value, option.textContent]);
+ assert.deepEqual(choices(), [
+ ["auto", "Legacy default"],
+ ["adaptive", "Auto"],
+ ["default", "Provider default"],
+ ["high", "High"],
+ ]);
+ select().value = "adaptive";
+ select().dispatchEvent(new dom.window.Event("change"));
+ assert.equal(dom.window.eval('settingsUI.collect("runtime").effortLevel'), "adaptive");
+ assert.equal(
+ choices().some(([value]) => value === "auto"),
+ false,
+ );
+ dom.window.eval('settingsUI.states.get("runtime").change("modelId", "b")');
+ assert.deepEqual(choices(), [["high", "High"]]);
+ assert.equal(dom.window.eval('settingsUI.collect("runtime").effortLevel'), "high");
+ dom.window.eval(
+ 'settingsUI.states.get("runtime").context.modelsByHarness.pi = [{id:"b"}]; settingsUI.states.get("runtime").changed()',
+ );
+ assert.deepEqual(choices(), [["high", "High"]]);
+ } finally {
+ dom.window.close();
+ }
+});
diff --git a/plugins/admin/ui/settings.ts b/plugins/admin/ui/settings.ts
index c3c61d5e0..bbd3b386f 100644
--- a/plugins/admin/ui/settings.ts
+++ b/plugins/admin/ui/settings.ts
@@ -19,7 +19,7 @@ import { html } from "lit";
import { classMap } from "lit/directives/class-map.js";
import { repeat } from "lit/directives/repeat.js";
-type Model = { id: string; name?: string };
+type Model = { id: string; name?: string; effortLevels?: string[] };
type Data = Record<string, any>;
export class SettingsState extends SettingState {
context: Data = {};
@@ -46,7 +46,12 @@ export class SettingsState extends SettingState {
return this.context.harnessOptions?.length ? this.context.harnessOptions : [this.context.harnessDefault || "pi"];
}
get efforts(): string[] {
- return this.context.thinkingLevelsByHarness?.[this.draft.harnessId] || ["auto"];
+ const advertised = this.models.find((model) => model.id === this.draft.modelId)?.effortLevels;
+ const levels: string[] = advertised ?? this.context.thinkingLevelsByHarness?.[this.draft.harnessId] ?? [];
+ const choices = levels.filter(
+ (level) => level !== "auto" && (advertised || (level !== "adaptive" && level !== "default")),
+ );
+ return this.draft.effortLevel === "auto" || !choices.length ? ["auto", ...choices] : choices;
}
get fastCapable() {
return (
@@ -149,7 +154,9 @@ const harnessLabels: Record<string, string> = {
claude: "Claude Code",
};
const effortLabels: Record<string, string> = {
- auto: "Auto",
+ auto: "Legacy default",
+ adaptive: "Auto",
+ default: "Provider default",
low: "Low",
medium: "Medium",
high: "High",
diff --git a/plugins/web-ui/src/composer-loadout.ts b/plugins/web-ui/src/composer-loadout.ts
index 808da1419..92d94a1b7 100644
--- a/plugins/web-ui/src/composer-loadout.ts
+++ b/plugins/web-ui/src/composer-loadout.ts
@@ -1,3 +1,5 @@
+import type { Api, Model } from "@earendil-works/pi-ai";
+import type { ModelMetadata } from "./pi-models.ts";
import { EFFORT_LEVELS, type EffortLevel, type ModelOption } from "./model-options.ts";
export interface LoadoutEntry {
@@ -68,13 +70,24 @@ export function reconcileLoadout(
return available.has(active.value) ? upsertLoadout(next, active) : next.slice(0, LOADOUT_CAP);
}
-export function effortLevelsForHarness(harnessId: string): Array<{ value: EffortLevel; label: string }> {
- return EFFORT_LEVELS.filter(({ value }) => {
+export function effortLevelsForHarness(
+ harnessId: string,
+ model?: Model<Api>,
+ selectedEffort?: string,
+): Array<{ value: EffortLevel; label: string }> {
+ const advertised = (model as ModelMetadata | undefined)?.effortLevelsByHarness?.[harnessId];
+ const levels = EFFORT_LEVELS.filter(({ value }) => {
+ if (value === "auto") return false;
+ if (advertised) return advertised.includes(value);
+ if (value === "adaptive" || value === "default") return false;
if (harnessId === "pi") return true;
if (harnessId === "claude") return value !== "ultracode";
if (harnessId === "codex") return value !== "max" && value !== "ultracode";
- return value === "auto";
- }).map((option) => ({ ...option, label: option.value === "xhigh" ? "Extra high" : option.label }));
+ return false;
+ });
+ return selectedEffort === "auto" || !levels.length
+ ? [EFFORT_LEVELS.find(({ value }) => value === "auto")!, ...levels]
+ : levels;
}
export function compatibleHarnessOptions<T extends { harnessId: string; model: { id: string } }>(
diff --git a/plugins/web-ui/src/composer.ts b/plugins/web-ui/src/composer.ts
index 465001555..c7a9cbeb7 100644
--- a/plugins/web-ui/src/composer.ts
+++ b/plugins/web-ui/src/composer.ts
@@ -224,8 +224,8 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface {
(restoredLoadout?.value === selected?.value ? restoredLoadout?.effort : undefined) ??
(getRuntimeConfig(scopeKey())?.effective.effortLevel as EffortLevel | undefined) ??
defaultEffortForModel(selected?.model);
- const levels = effortLevelsForHarness(selected?.harnessId ?? "");
- return levels.some((level) => level.value === effort) ? effort : "auto";
+ const levels = effortLevelsForHarness(selected?.harnessId ?? "", selected?.model, effort);
+ return levels.some((level) => level.value === effort) ? effort : levels[0]!.value;
},
set effortLevel(value: EffortLevel) {
++effortSelectionRevision;
@@ -947,9 +947,9 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface {
}
function normalizeLoadoutEntry(entry: LoadoutEntry, option: ModelOption): LoadoutEntry {
- const levels = effortLevelsForHarness(option.harnessId);
+ const levels = effortLevelsForHarness(option.harnessId, option.model, entry.effort);
const defaultEffort = defaultEffortForModel(option.model);
- const fallbackEffort = levels.some((level) => level.value === defaultEffort) ? defaultEffort : "auto";
+ const fallbackEffort = levels.some((level) => level.value === defaultEffort) ? defaultEffort : levels[0]!.value;
return {
value: option.value,
effort: levels.some((level) => level.value === entry.effort) ? entry.effort : fallbackEffort,
@@ -1758,7 +1758,13 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface {
function selectEffort(level: EffortLevel, agent: Agent): void {
const selected = currentModelOption();
- if (!selected || !effortLevelsForHarness(selected.harnessId).some((option) => option.value === level)) return;
+ if (
+ !selected ||
+ !effortLevelsForHarness(selected.harnessId, selected.model, composerState.effortLevel).some(
+ (option) => option.value === level,
+ )
+ )
+ return;
composerState.effortLevel = level;
rememberActiveTweaks(selected);
ctx.chat.drawActiveChat(agent);
diff --git a/plugins/web-ui/src/context-model.ts b/plugins/web-ui/src/context-model.ts
index f53aa8558..b98d27b9f 100644
--- a/plugins/web-ui/src/context-model.ts
+++ b/plugins/web-ui/src/context-model.ts
@@ -101,9 +101,13 @@ function selectedValue(config: RuntimeConfig): string {
return config.scopeOverride ? `${config.scopeOverride.harnessId}:${config.scopeOverride.modelId}` : INHERIT;
}
-function effortLevelsFor(harnessId: string): Array<{ value: EffortLevel; label: string }> {
+function effortLevelsFor(
+ harnessId: string,
+ model?: ModelOption["model"],
+ effort?: string,
+): Array<{ value: EffortLevel; label: string }> {
if (!harnessSupportsEffort(harnessId)) return [];
- return effortLevelsForHarness(harnessId);
+ return effortLevelsForHarness(harnessId, model, effort);
}
function selectedEffort(config: RuntimeConfig): string {
@@ -131,6 +135,9 @@ async function choose(scope: string, value: string, effort?: string, fast = fals
try {
const sep = value.indexOf(":");
const harnessId = value.slice(0, sep);
+ const model = contextModelState.config
+ ? optionsFor(contextModelState.config).find((option) => option.value === value)?.model
+ : undefined;
await saveRuntimeConfig(
scope,
value === INHERIT
@@ -139,7 +146,9 @@ async function choose(scope: string, value: string, effort?: string, fast = fals
harnessId,
modelId: value.slice(sep + 1),
fastMode: fast && harnessSupportsFastMode(harnessId) && modelSupportsFastMode(scope, value.slice(sep + 1)),
- ...(effort && effortLevelsFor(harnessId).some((o) => o.value === effort) ? { effortLevel: effort } : {}),
+ ...(effort && effortLevelsFor(harnessId, model, effort).some((o) => o.value === effort)
+ ? { effortLevel: effort }
+ : {}),
},
);
if (seq !== loadSeq) return;
@@ -178,11 +187,10 @@ function contextPicker(scopeId: string) {
if (contextModelState.saving) return;
const option = options().find((option) => option.value === entry.value);
if (!option) return;
+ const levels = effortLevelsForHarness(option.harnessId, option.model, entry.effort);
const normalized = {
...entry,
- effort: effortLevelsForHarness(option.harnessId).some((level) => level.value === entry.effort)
- ? entry.effort
- : ("auto" as EffortLevel),
+ effort: levels.some((level) => level.value === entry.effort) ? entry.effort : levels[0]!.value,
fast: entry.fast && harnessSupportsFastMode(option.harnessId) && modelSupportsFastMode(scopeId, option.model.id),
};
void choose(scopeId, normalized.value, normalized.effort, normalized.fast);
diff --git a/plugins/web-ui/src/model-picker.ts b/plugins/web-ui/src/model-picker.ts
index 0d892929d..f3a12f27a 100644
--- a/plugins/web-ui/src/model-picker.ts
+++ b/plugins/web-ui/src/model-picker.ts
@@ -273,7 +273,7 @@ export function createModelPicker<T>(bindings: ModelPickerBindings<T>) {
</button>
${
effort
- ? effortLevelsForHarness(selected.harnessId).map(
+ ? effortLevelsForHarness(selected.harnessId, selected.model, composerState.effortLevel).map(
(level) =>
html` <button
class="loadout-effort"
diff --git a/plugins/web-ui/src/pi-models.ts b/plugins/web-ui/src/pi-models.ts
index 8d383ae99..01c7553b8 100644
--- a/plugins/web-ui/src/pi-models.ts
+++ b/plugins/web-ui/src/pi-models.ts
@@ -8,6 +8,7 @@ export type ModelMetadata = Pick<
label: string;
buttonLabel: string;
fastMode: boolean;
+ effortLevelsByHarness?: Record<string, string[]>;
};
export function getBaseModel(id: string, metadata?: ModelMetadata): Model<Api> {
diff --git a/plugins/web-ui/src/runtime-capabilities.ts b/plugins/web-ui/src/runtime-capabilities.ts
index 4544f13a2..299aa1197 100644
--- a/plugins/web-ui/src/runtime-capabilities.ts
+++ b/plugins/web-ui/src/runtime-capabilities.ts
@@ -1,9 +1,11 @@
import type { Api, Model } from "@earendil-works/pi-ai";
-export type EffortLevel = "low" | "medium" | "high" | "xhigh" | "max" | "ultracode" | "auto";
+export type EffortLevel = "low" | "medium" | "high" | "xhigh" | "max" | "ultracode" | "auto" | "adaptive" | "default";
export const EFFORT_LEVELS: Array<{ value: EffortLevel; label: string }> = [
- { value: "auto", label: "Auto" },
+ { value: "auto", label: "Legacy default" },
+ { value: "adaptive", label: "Auto" },
+ { value: "default", label: "Provider default" },
{ value: "low", label: "Low" },
{ value: "medium", label: "Medium" },
{ value: "high", label: "High" },
diff --git a/plugins/web-ui/test/composer-loadout.test.ts b/plugins/web-ui/test/composer-loadout.test.ts
index 7c7c77e23..e1634414a 100644
--- a/plugins/web-ui/test/composer-loadout.test.ts
+++ b/plugins/web-ui/test/composer-loadout.test.ts
@@ -198,18 +198,51 @@ test("the model catalog falls back from unavailable saved harnesses using only c
test("harness effort choices exclude unsupported settings and label extra high clearly", () => {
assert.deepEqual(
effortLevelsForHarness("pi").map(({ value }) => value),
- ["auto", "low", "medium", "high", "xhigh", "max", "ultracode"],
+ ["low", "medium", "high", "xhigh", "max", "ultracode"],
);
assert.deepEqual(
effortLevelsForHarness("claude").map(({ value }) => value),
- ["auto", "low", "medium", "high", "xhigh", "max"],
+ ["low", "medium", "high", "xhigh", "max"],
);
assert.deepEqual(
effortLevelsForHarness("codex").map(({ value }) => value),
- ["auto", "low", "medium", "high", "xhigh"],
+ ["low", "medium", "high", "xhigh"],
);
for (const harnessId of ["pi", "claude", "codex"])
assert.equal(effortLevelsForHarness(harnessId).find(({ value }) => value === "xhigh")?.label, "Extra high");
for (const harnessId of ["opencode", "mock", "unknown"])
- assert.deepEqual(effortLevelsForHarness(harnessId), [{ value: "auto", label: "Auto" }]);
+ assert.deepEqual(effortLevelsForHarness(harnessId), [{ value: "auto", label: "Legacy default" }]);
+});
+
+test("native reasoning choices require model and harness metadata while legacy settings survive", () => {
+ const model = {
+ ...option("pi:one").model,
+ effortLevelsByHarness: {
+ pi: ["auto", "adaptive", "default", "low", "high"],
+ claude: ["auto", "low", "high"],
+ },
+ };
+ assert.deepEqual(effortLevelsForHarness("pi", model), [
+ { value: "adaptive", label: "Auto" },
+ { value: "default", label: "Provider default" },
+ { value: "low", label: "Low" },
+ { value: "high", label: "High" },
+ ]);
+ assert.equal(effortLevelsForHarness("pi", model, "auto")[0]?.label, "Legacy default");
+ for (const harness of ["claude", "codex"])
+ assert.ok(
+ effortLevelsForHarness(harness, model, "adaptive").every(
+ ({ value }) => value !== "adaptive" && value !== "default",
+ ),
+ );
+ for (const selected of ["auto", "adaptive", "default"])
+ assert.ok(
+ effortLevelsForHarness("pi", option("pi:one").model, selected).every(
+ ({ value }) => value !== "adaptive" && value !== "default",
+ ),
+ );
+ const withoutEfforts = { ...model, effortLevelsByHarness: { pi: [] } };
+ assert.deepEqual(effortLevelsForHarness("pi", withoutEfforts), [{ value: "auto", label: "Legacy default" }]);
+ const saved = [entry("pi:one", "adaptive"), entry("pi:two", "default"), entry("pi:three", "auto")];
+ assert.deepEqual(parseLoadout(JSON.stringify(saved)), saved);
});
diff --git a/plugins/web-ui/test/composer-picker.test.ts b/plugins/web-ui/test/composer-picker.test.ts
index 003b75640..8a073e610 100644
--- a/plugins/web-ui/test/composer-picker.test.ts
+++ b/plugins/web-ui/test/composer-picker.test.ts
@@ -642,6 +642,46 @@ test("the personal-account picker preserves composer choices and saves context d
fastMode: false,
});
assert.equal(context.contextModelState.config.effective.modelId, "beta");
+ config.modelCatalog!.alpha!.effortLevelsByHarness = {
+ pi: ["auto", "adaptive", "default", "low", "high"],
+ claude: ["auto", "low", "high"],
+ };
+ config.effective = { harnessId: "pi", modelId: "alpha", effortLevel: "auto" };
+ ctx.chat.state.threadRef = "web:tester:native-auto";
+ localStorage.removeItem("web-ui:loadout");
+ await composer!.refreshRuntimeSelection(null, agent, true);
+ await mount();
+ button('[data-loadout-section="effort"]').click();
+ const effortChoices = () => [...host.querySelectorAll<HTMLButtonElement>(".loadout-effort")];
+ assert.deepEqual(
+ effortChoices().map((item) => item.textContent?.trim()),
+ ["Legacy default", "Auto", "Provider default", "Low", "High"],
+ );
+ effortChoices()
+ .find((item) => item.textContent?.trim() === "Auto")!
+ .click();
+ assert.equal(composer!.state.effortLevel, "adaptive");
+ assert.equal(saved().find(({ value }) => value === "pi:alpha")?.effort, "adaptive");
+ button('[data-loadout-section="effort"]').click();
+ assert.equal(
+ effortChoices().some((item) => item.textContent?.trim() === "Legacy default"),
+ false,
+ );
+ effortChoices()
+ .find((item) => item.textContent?.trim() === "Provider default")!
+ .click();
+ assert.equal(composer!.state.effortLevel, "default");
+ context.resetContextModel();
+ seedRuntimeConfig(config.scopeId, { ...config });
+ await context.loadContextModel(config.scopeId, drawContext);
+ contextButton(".loadout-button").click();
+ contextButton('[data-loadout-section="effort"]').click();
+ [...contextHost.querySelectorAll<HTMLButtonElement>(".loadout-effort")]
+ .find((item) => item.textContent?.trim() === "Auto")!
+ .click();
+ await tick();
+ assert.equal(updates.at(-1)?.effortLevel, "adaptive");
+ assert.equal(context.contextModelState.config.effective.effortLevel, "adaptive");
} finally {
resetContext?.();
composer?.dispose();
diff --git a/plugins/web-ui/test/composer-source.test.ts b/plugins/web-ui/test/composer-source.test.ts
index 42d31cd6b..f0ef8b52f 100644
--- a/plugins/web-ui/test/composer-source.test.ts
+++ b/plugins/web-ui/test/composer-source.test.ts
@@ -43,7 +43,7 @@ test("switching setups preserves prior tweaks and validates effort and Fast for
composer.indexOf("function normalizeLoadoutEntry"),
composer.indexOf("function seededLoadout"),
);
- assert.ok(normalize.includes("effortLevelsForHarness(option.harnessId)"));
+ assert.ok(normalize.includes("effortLevelsForHarness(option.harnessId, option.model, entry.effort)"));
assert.ok(/levels.some\([\s\S]*?\? entry.effort/.test(normalize));
assert.ok(apply.includes("normalizeLoadoutEntry(entry, option)"));
assert.ok(
diff --git a/plugins/web-ui/test/context-model-source.test.ts b/plugins/web-ui/test/context-model-source.test.ts
index 334b793de..f2492bf94 100644
--- a/plugins/web-ui/test/context-model-source.test.ts
+++ b/plugins/web-ui/test/context-model-source.test.ts
@@ -45,7 +45,7 @@ test("an in-flight pick wins the saving re-render — no snap-back while the sav
});
test("context effort and Fast use the shared harness and model capabilities", () => {
- assert.match(panel, /return effortLevelsForHarness\(harnessId\)/);
+ assert.match(panel, /return effortLevelsForHarness\(harnessId, model, effort\)/);
assert.match(panel, /harnessSupportsFastMode\(harnessId\)/);
assert.match(panel, /modelSupportsFastMode\(scope, value.slice\(sep \+ 1\)\)/);
});
diff --git a/plugins/web-ui/test/pi-models.test.ts b/plugins/web-ui/test/pi-models.test.ts
index 66189bccc..f24f18dca 100644
--- a/plugins/web-ui/test/pi-models.test.ts
+++ b/plugins/web-ui/test/pi-models.test.ts
@@ -12,7 +12,13 @@ test("models require server metadata even when the browser SDK knows their id",
test("browser model preserves safe server geometry and does not share mutable metadata", () => {
const spec = metadata("future", "Future", "anthropic");
+ spec.effortLevelsByHarness = { pi: ["auto", "adaptive", "default", "high"] };
const model = getBaseModel("future", spec);
+ const capabilities = (model as typeof model & { effortLevelsByHarness: Record<string, string[]> })
+ .effortLevelsByHarness;
+ assert.deepEqual(capabilities, spec.effortLevelsByHarness);
+ capabilities.pi!.push("low");
+ assert.equal(spec.effortLevelsByHarness.pi!.includes("low"), false);
assert.equal(model.api, "anthropic-messages");
assert.equal(model.contextWindow, spec.contextWindow);
model.cost.input = 999;
diff --git a/src/api/routes/admin-resources.ts b/src/api/routes/admin-resources.ts
index 2606a44f1..0baf5e649 100644
--- a/src/api/routes/admin-resources.ts
+++ b/src/api/routes/admin-resources.ts
@@ -644,7 +644,7 @@ export const ADMIN_RESOURCES: readonly AdminResource[] = [
if (!approved.includes(harnessId)) return { error: `harness ${harnessId} is not approved` };
if (typeof modelId !== "string" || !modelSupportedByHarness(modelId, harnessId))
return { error: `model ${String(modelId)} is not supported by ${harnessId}` };
- const thinkingLevels = thinkingLevelsForHarness(harnessId);
+ const thinkingLevels = thinkingLevelsForHarness(harnessId, modelId);
if (typeof effortLevel !== "string" || !thinkingLevels.includes(effortLevel))
return { error: `runtime requires effortLevel (${thinkingLevels.join(" | ")}) for ${harnessId}` };
if (typeof fastMode !== "boolean") return { error: "runtime requires fastMode (boolean)" };
diff --git a/src/api/routes/admin/scope-config.ts b/src/api/routes/admin/scope-config.ts
index 29e835f4c..26f442bb9 100644
--- a/src/api/routes/admin/scope-config.ts
+++ b/src/api/routes/admin/scope-config.ts
@@ -375,7 +375,12 @@ async function scopeModelOptions(deps: ApiCtx["deps"], values: Record<string, un
harnessOptions: HARNESS_IDS.filter(
(id) => id !== "mock" && (approvedHarnesses.includes(id) || runtime?.harnessId === id),
),
- modelsByHarness: Object.fromEntries(HARNESS_IDS.map((id) => [id, modelsFor(id)])),
+ modelsByHarness: Object.fromEntries(
+ HARNESS_IDS.map((id) => [
+ id,
+ modelsFor(id).map((model) => ({ ...model, effortLevels: thinkingLevelsForHarness(id, model.id) })),
+ ]),
+ ),
thinkingLevelsByHarness: Object.fromEntries(
HARNESS_IDS.filter((id) => id !== "mock").map((id) => [id, thinkingLevelsForHarness(id)]),
),
diff --git a/src/api/routes/surface.ts b/src/api/routes/surface.ts
index 5d89133e9..8bc41fb63 100644
--- a/src/api/routes/surface.ts
+++ b/src/api/routes/surface.ts
@@ -11,7 +11,7 @@ import {
isHarnessId,
modelSupportedByHarness,
modelOfferedInWebui,
- THINKING_LEVELS,
+ thinkingLevelsForHarness,
fastModeModelIds,
} from "../../model/pi-models.ts";
import { builtInModelCatalog, selectableCatalogForHarness, selectableModelCatalog } from "../../model/model-catalog.ts";
@@ -1269,7 +1269,7 @@ async function putRuntimeConfig(ctx: ApiCtx): Promise<void> {
return sendJson(ctx.res, 400, { error: "model_not_supported" });
if (!(await webuiModelEnabled(ctx, modelId))) return sendJson(ctx.res, 400, { error: "model_not_enabled" });
const effortLevel = ctx.body.effortLevel ?? "auto";
- if (typeof effortLevel !== "string" || !(THINKING_LEVELS as readonly string[]).includes(effortLevel))
+ if (typeof effortLevel !== "string" || !thinkingLevelsForHarness(harnessId, modelId).includes(effortLevel))
return sendJson(ctx.res, 400, { error: "effort_not_supported" });
const fastMode = ctx.body.fastMode ?? false;
if (typeof fastMode !== "boolean") return sendJson(ctx.res, 400, { error: "fast_mode_invalid" });
diff --git a/src/api/runtime-config.ts b/src/api/runtime-config.ts
index 43b7ba3dd..563d0bc86 100644
--- a/src/api/runtime-config.ts
+++ b/src/api/runtime-config.ts
@@ -72,7 +72,9 @@ export async function userRuntimeConfigBody(ctx: { deps: RuntimeDeps }, scope: S
...snapshot.effective,
harnessId: route.harness,
modelId: route.model,
- effortLevel: thinkingLevelsForHarness(route.harness).includes(snapshot.effective.effortLevel ?? "auto")
+ effortLevel: thinkingLevelsForHarness(route.harness, route.model).includes(
+ snapshot.effective.effortLevel ?? "auto",
+ )
? snapshot.effective.effortLevel
: "auto",
fastMode:
@@ -217,7 +219,10 @@ export async function runtimeConfigBody(
export function validateRuntimeChoice(choice: RuntimeChoice): string | null {
if (!modelSupportedByHarness(choice.modelId, choice.harnessId)) return "model_not_supported";
- if (choice.effortLevel !== undefined && !thinkingLevelsForHarness(choice.harnessId).includes(choice.effortLevel))
+ if (
+ choice.effortLevel !== undefined &&
+ !thinkingLevelsForHarness(choice.harnessId, choice.modelId).includes(choice.effortLevel)
+ )
return "effort_not_supported";
if (choice.fastMode !== undefined && typeof choice.fastMode !== "boolean") return "fast_mode_invalid";
if (choice.fastMode && (!harnessSupportsFastMode(choice.harnessId) || !fastModeModelIds().includes(choice.modelId)))
@@ -249,5 +254,8 @@ export async function availableRuntimeError(
!(await webuiModelEnabled(ctx, choice.modelId))
)
return "runtime is no longer available or enabled on this deployment";
- return validateRuntimeChoice({ ...choice, effortLevel: undefined });
+ return validateRuntimeChoice({
+ ...choice,
+ effortLevel: choice.effortLevel === "adaptive" || choice.effortLevel === "default" ? choice.effortLevel : undefined,
+ });
}
diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.ts
index 8a9a182f6..127b1ddbb 100644
--- a/src/harness/agent-tools.ts
+++ b/src/harness/agent-tools.ts
@@ -4022,7 +4022,12 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
action: Type.Union([Type.Literal("get"), Type.Literal("set"), Type.Literal("inherit")]),
model: Type.Optional(Type.String({ description: "Model ID or exact display name from get, such as Astra." })),
harness: Type.Optional(Type.String()),
- effort: Type.Optional(Type.String()),
+ effort: Type.Optional(
+ Type.String({
+ description:
+ "Use modelCatalog[modelId].effortLevelsByHarness[harnessId] from get. adaptive = native Auto; default = provider default; auto = legacy harness default, not adaptive reasoning.",
+ }),
+ ),
fastMode: Type.Optional(Type.Boolean()),
lifetime: Type.Optional(Type.Union([Type.Literal("task"), Type.Literal("scope")])),
}),
diff --git a/src/harness/harness-router.ts b/src/harness/harness-router.ts
index cd0889ea0..b001da334 100644
--- a/src/harness/harness-router.ts
+++ b/src/harness/harness-router.ts
@@ -98,10 +98,17 @@ async function runTurnEnforcingGoal(
}
function normalizeRuntimeChoice(choice: RuntimeChoice): RuntimeChoice {
+ if (
+ (choice.effortLevel === "adaptive" || choice.effortLevel === "default") &&
+ !thinkingLevelsForHarness(choice.harnessId, choice.modelId).includes(choice.effortLevel)
+ )
+ throw new NonRetryableTurnError(
+ `${choice.effortLevel} reasoning is not supported by ${choice.harnessId}/${choice.modelId}`,
+ );
return {
harnessId: choice.harnessId,
modelId: choice.modelId,
- ...(choice.effortLevel && thinkingLevelsForHarness(choice.harnessId).includes(choice.effortLevel)
+ ...(choice.effortLevel && thinkingLevelsForHarness(choice.harnessId, choice.modelId).includes(choice.effortLevel)
? { effortLevel: choice.effortLevel }
: {}),
...(typeof choice.fastMode === "boolean"
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index 7e41d3187..acd53789b 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -37,6 +37,8 @@ const TURN_EFFORT_LEVELS = new Set<string>([
"max",
"ultracode",
"auto",
+ "default",
+ "adaptive",
]);
import type { ConversationTurn, ScopeId, SessionEntry } from "../types.ts";
import type {
@@ -58,6 +60,8 @@ import {
auxiliaryModelForProvider,
defaultModelForHarness,
defaultInteractiveThinkingLevel,
+ modelSupportsAdaptiveThinking,
+ modelSupportsProviderDefault,
modelDisplayName,
resolveModel,
getRequiredModel,
@@ -415,7 +419,7 @@ export function sanitizeTitle(out = ""): string | undefined {
interface TurnSession {
agentSession: AgentSession;
- ref: ToolContextRef;
+ ref: ToolContextRef & { effortLevel?: string };
composedPromptTokens: number;
cwd: string;
agentDir: string;
@@ -1486,6 +1490,10 @@ export function withRequestHeaders(model: Model<Api>, direct: boolean, fast: boo
export function applyTurnEffort(session: AgentSession, level?: string): void {
if (!level || !TURN_EFFORT_LEVELS.has(level)) return;
+ if (level === "adaptive" || level === "default") {
+ session.setThinkingLevel("off");
+ return;
+ }
const effectiveLevel =
level === "auto" && session.state.model ? defaultInteractiveThinkingLevel(session.state.model) : level;
const normalizedLevel = effectiveLevel === "auto" ? "medium" : effectiveLevel;
@@ -1495,6 +1503,27 @@ export function applyTurnEffort(session: AgentSession, level?: string): void {
session.setThinkingLevel(providerLevel as ModelThinkingLevel);
}
+export function applyReasoningMode<T>(payload: T, model: Model<Api>, level?: string): T {
+ if (level !== "adaptive" && level !== "default") return payload;
+ if (level === "adaptive" ? !modelSupportsAdaptiveThinking(model) : !modelSupportsProviderDefault(model))
+ throw new NonRetryableTurnError(`${level} reasoning is not supported by ${model.id}`);
+ if (!payload || typeof payload !== "object") return payload;
+ const body = payload as Record<string, unknown>;
+ delete body.thinking;
+ delete body.reasoning;
+ delete body.reasoning_effort;
+ if (body.output_config && typeof body.output_config === "object") {
+ const outputConfig = { ...body.output_config } as Record<string, unknown>;
+ delete outputConfig.effort;
+ if (Object.keys(outputConfig).length) body.output_config = outputConfig;
+ else delete body.output_config;
+ }
+ if (model.reasoning && ["openai-responses", "openai-codex-responses"].includes(model.api))
+ body.include = [...new Set([...(Array.isArray(body.include) ? body.include : []), "reasoning.encrypted_content"])];
+ if (level === "adaptive") body.thinking = { type: "adaptive", display: "summarized" };
+ return payload;
+}
+
export function createPiHarness(opts?: PiHarnessOptions): Harness {
const configuredModelId = opts?.modelId;
const resolveModelId = (scope?: ScopeId): string =>
@@ -1591,7 +1620,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
turnProviderKeys ? undefined : modelGateway,
systemCacheSplit ? "long" : undefined,
);
- const ref: ToolContextRef = { current: null };
+ const ref: TurnSession["ref"] = { current: null };
const { resourceLoader, settingsManager, cwd, agentDir, ephemeralCwd } = await createIsolatedResources(
tempDirPrefix,
composedPrompt,
@@ -1676,6 +1705,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
ref.pendingPrepareNextTurn = undefined;
ref.pendingTransformContext = undefined;
applyFastSpeed(payload, ref.fast, (model as { api?: string } | undefined)?.api);
+ applyReasoningMode(payload, model as Model<Api>, ref.effortLevel);
const result = prior ? await prior(payload, model) : payload;
const capturedPayload = captureRequests ? sanitizeLlmPayload(result ?? payload, model) : undefined;
let finalPayload = await withDocumentInputs(
@@ -1815,7 +1845,8 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
const defaultThinkingLevel = entry.agentSession.model
? defaultInteractiveThinkingLevel(entry.agentSession.model)
: "auto";
- applyTurnEffort(entry.agentSession, turn.runtime?.effortLevel ?? defaultThinkingLevel);
+ entry.ref.effortLevel = turn.runtime?.effortLevel ?? defaultThinkingLevel;
+ applyTurnEffort(entry.agentSession, entry.ref.effortLevel);
const toolWallByStep: number[][] = [];
const gapWork: GapWork[] = [];
@@ -2207,7 +2238,8 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
withRequestHeaders(fallback, !turnModelGateway?.models[fallbackId], wantFast),
);
entry.ref.fast = wantFast;
- applyTurnEffort(entry.agentSession, turn.runtime?.effortLevel ?? defaultInteractiveThinkingLevel(fallback));
+ entry.ref.effortLevel = turn.runtime?.effortLevel ?? defaultInteractiveThinkingLevel(fallback);
+ applyTurnEffort(entry.agentSession, entry.ref.effortLevel);
const state = entry.agentSession.agent.state;
for (let i = state.messages.length - 1; i >= messagesBefore; i--) {
const m = state.messages[i] as { role?: string; stopReason?: string } | undefined;
diff --git a/src/model/pi-models.ts b/src/model/pi-models.ts
index 6da2639bf..98e1923bd 100644
--- a/src/model/pi-models.ts
+++ b/src/model/pi-models.ts
@@ -24,15 +24,48 @@ export function codexSubscriptionModelId(id: string): string {
export function codexProviderModelId(id: string): string {
return id.startsWith(CODEX_SUBSCRIPTION_PREFIX) ? id.slice(CODEX_SUBSCRIPTION_PREFIX.length) : id;
}
-export const THINKING_LEVELS = ["auto", "low", "medium", "high", "xhigh", "max", "ultracode"] as const;
+export const THINKING_LEVELS = [
+ "auto",
+ "default",
+ "adaptive",
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max",
+ "ultracode",
+] as const;
export const HARNESS_IDS = ["pi", "opencode", "codex", "claude", "mock"] as const;
export type HarnessId = (typeof HARNESS_IDS)[number];
-export function thinkingLevelsForHarness(harnessId: HarnessId): readonly string[] {
- if (harnessId === "pi") return THINKING_LEVELS;
- if (harnessId === "claude") return THINKING_LEVELS.filter((level) => level !== "ultracode");
- if (harnessId === "codex") return THINKING_LEVELS.filter((level) => level !== "max" && level !== "ultracode");
- return ["auto"];
+export function modelSupportsAdaptiveThinking(model: Pick<Model<Api>, "api" | "reasoning" | "compat">): boolean {
+ return (
+ model.api === "anthropic-messages" &&
+ model.reasoning &&
+ (model.compat as { forceAdaptiveThinking?: boolean } | undefined)?.forceAdaptiveThinking === true
+ );
+}
+
+export function modelSupportsProviderDefault(model: Pick<Model<Api>, "api" | "compat">): boolean {
+ return (
+ ["anthropic-messages", "openai-responses", "openai-codex-responses"].includes(model.api) ||
+ (model.api === "openai-completions" &&
+ (model.compat as { thinkingFormat?: string } | undefined)?.thinkingFormat === "openai")
+ );
+}
+
+export function thinkingLevelsForHarness(harnessId: HarnessId, modelId?: string): readonly string[] {
+ const model = modelId ? resolveModel(modelId) : undefined;
+ return THINKING_LEVELS.filter((level) => {
+ if (level === "adaptive")
+ return harnessId === "pi" && (!modelId || (!!model && modelSupportsAdaptiveThinking(model)));
+ if (level === "default")
+ return harnessId === "pi" && (!modelId || (!!model && modelSupportsProviderDefault(model)));
+ if (harnessId === "pi") return true;
+ if (harnessId === "claude") return level !== "ultracode";
+ if (harnessId === "codex") return level !== "max" && level !== "ultracode";
+ return level === "auto";
+ });
}
export function harnessSupportsFastMode(harnessId: HarnessId): boolean {
@@ -624,5 +657,8 @@ export function safeModelMetadata(id: string) {
maxTokens: model.maxTokens,
cost: structuredClone(model.cost),
fastMode: modelSupportsFastMode(id),
+ effortLevelsByHarness: Object.fromEntries(
+ HARNESS_IDS.map((harness) => [harness, thinkingLevelsForHarness(harness, id)]),
+ ),
};
}
diff --git a/test/model-credential-route.test.ts b/test/model-credential-route.test.ts
index 313342ed9..447f03ee2 100644
--- a/test/model-credential-route.test.ts
+++ b/test/model-credential-route.test.ts
@@ -412,7 +412,14 @@ test("admin scope keeps the selected runtime model visible when its provider is
};
assert.deepEqual(data.runtime, { harnessId: "pi", modelId: "claude-opus-5", orgRevision: 1, revision: 1 });
assert.deepEqual(data.harnessOptions, ["pi"]);
- assert.deepEqual(data.modelsByHarness.pi, [{ id: "claude-opus-5", name: "Claude Opus 5", provider: "anthropic" }]);
+ assert.deepEqual(data.modelsByHarness.pi, [
+ {
+ id: "claude-opus-5",
+ name: "Claude Opus 5",
+ provider: "anthropic",
+ effortLevels: ["auto", "default", "adaptive", "low", "medium", "high", "xhigh", "max", "ultracode"],
+ },
+ ]);
} finally {
await srv.close();
}
diff --git a/test/native-auto-reasoning.test.ts b/test/native-auto-reasoning.test.ts
new file mode 100644
index 000000000..effb9fe46
--- /dev/null
+++ b/test/native-auto-reasoning.test.ts
@@ -0,0 +1,209 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { applyReasoningMode, createPiHarness } from "../src/harness/pi-harness.ts";
+import { getRequiredModel, thinkingLevelsForHarness, safeModelMetadata } from "../src/model/pi-models.ts";
+import { validateRuntimeChoice } from "../src/api/runtime-config.ts";
+import type { HarnessTurnInput } from "../src/harness/harness.ts";
+import { setGatewayModels } from "../src/model/gateway-models.ts";
+
+const adaptiveId = "claude-sonnet-4-6";
+
+test("native Auto is model and harness specific, while legacy auto remains valid", () => {
+ for (const harnessId of ["pi", "codex", "claude", "opencode"] as const) {
+ const modelId = harnessId === "codex" ? "gpt-5.4" : adaptiveId;
+ assert.equal(validateRuntimeChoice({ harnessId, modelId, effortLevel: "auto" }), null);
+ assert.equal(
+ validateRuntimeChoice({ harnessId, modelId, effortLevel: "adaptive" }),
+ harnessId === "pi" ? null : "effort_not_supported",
+ );
+ }
+ for (const modelId of ["gpt-6-astra", "codex/gpt-5.4", "claude-haiku-4-5", "openrouter/auto"])
+ assert.equal(validateRuntimeChoice({ harnessId: "pi", modelId, effortLevel: "adaptive" }), "effort_not_supported");
+ assert.deepEqual(safeModelMetadata(adaptiveId)?.effortLevelsByHarness.pi, thinkingLevelsForHarness("pi", adaptiveId));
+});
+
+test("gateway capability is determined by the real API and metadata, not its model name", (t) => {
+ t.after(() => setGatewayModels([]));
+ const model = getRequiredModel(adaptiveId);
+ setGatewayModels([
+ { ...model, provider: "qm:gateway", id: "gateway/opaque", api: "anthropic-messages" },
+ { ...model, provider: "qm:gateway", id: "gateway/claude-sonnet-4-6", api: "openai-completions", compat: {} },
+ ]);
+ assert.ok(thinkingLevelsForHarness("pi", "gateway/opaque").includes("adaptive"));
+ assert.ok(!thinkingLevelsForHarness("pi", "gateway/claude-sonnet-4-6").includes("adaptive"));
+});
+
+test("mode serialization retains unrelated output configuration and rejects unsupported Auto", () => {
+ const payload = {
+ thinking: { type: "enabled", budget_tokens: 1000 },
+ output_config: { effort: "high", format: "json" },
+ };
+ assert.deepEqual(applyReasoningMode(payload, getRequiredModel(adaptiveId), "adaptive"), {
+ thinking: { type: "adaptive", display: "summarized" },
+ output_config: { format: "json" },
+ });
+ assert.throws(() => applyReasoningMode({}, getRequiredModel("gpt-5.4"), "adaptive"), /not supported/);
+});
+
+test("real Pi AgentSession and Anthropic serializer send Auto without a fixed effort across credential routes", async (t) => {
+ const originalFetch = globalThis.fetch;
+ t.after(() => {
+ globalThis.fetch = originalFetch;
+ });
+ const requests: Array<{ headers: Headers; body: Record<string, any> }> = [];
+ let sendTool = false;
+ globalThis.fetch = (async (_url: unknown, init?: RequestInit) => {
+ requests.push({ headers: new Headers(init?.headers), body: JSON.parse(String(init?.body)) });
+ const events: Array<Record<string, unknown>> = [
+ {
+ type: "message_start",
+ message: {
+ id: "msg_test",
+ type: "message",
+ role: "assistant",
+ model: adaptiveId,
+ content: [],
+ usage: { input_tokens: 1, output_tokens: 0 },
+ },
+ },
+ { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
+ { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "OK" } },
+ { type: "content_block_stop", index: 0 },
+ { type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 1 } },
+ { type: "message_stop" },
+ ];
+ if (sendTool) {
+ sendTool = false;
+ events.splice(
+ 1,
+ 4,
+ { type: "content_block_start", index: 0, content_block: { type: "thinking", thinking: "", signature: "" } },
+ {
+ type: "content_block_delta",
+ index: 0,
+ delta: { type: "thinking_delta", thinking: "Use the attachment tool." },
+ },
+ {
+ type: "content_block_delta",
+ index: 0,
+ delta: { type: "signature_delta", signature: "test-preserved-signature" },
+ },
+ { type: "content_block_stop", index: 0 },
+ {
+ type: "content_block_start",
+ index: 1,
+ content_block: { type: "tool_use", id: "attach-native-auto", name: "attach", input: {} },
+ },
+ {
+ type: "content_block_delta",
+ index: 1,
+ delta: { type: "input_json_delta", partial_json: JSON.stringify({ files: ["test.txt"] }) },
+ },
+ { type: "content_block_stop", index: 1 },
+ );
+ events[events.length - 2] = {
+ type: "message_delta",
+ delta: { stop_reason: "tool_use", stop_sequence: null },
+ usage: { output_tokens: 10 },
+ };
+ }
+ return new Response(events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""), {
+ headers: { "content-type": "text/event-stream" },
+ });
+ }) as typeof fetch;
+ for (const modelId of [adaptiveId, "claude-opus-5-5"])
+ for (const route of ["api-key", "personal-key", "oauth", "gateway"] as const) {
+ await t.test(`${modelId}/${route}`, async () => {
+ const harness = createPiHarness({
+ defaultModelId: modelId,
+ ...(route === "gateway"
+ ? {
+ modelGateway: {
+ url: "https://gateway.invalid",
+ apiKey: "test-key",
+ apiKeyHeader: "x-gateway-key",
+ models: { [modelId]: "routed-model" },
+ },
+ }
+ : { apiKey: "sk-offline-test-key" }),
+ });
+ t.after(() => harness.turns.close?.());
+ for (const effortLevel of ["high", "adaptive", "default", "auto", "adaptive", "low"]) {
+ sendTool = true;
+ const result = await harness.turns.runTurn({
+ session: { id: `native-auto-${modelId}-${route}` } as HarnessTurnInput["session"],
+ input: "Reply OK",
+ systemPrompt: "Reply OK",
+ history: [],
+ tools: { attach: async () => ({ ok: true, files: [], staged: 0 }) } as unknown as HarnessTurnInput["tools"],
+ scopeLabel: "personal:test",
+ orgScopeId: "org:test",
+ runtime: { effortLevel },
+ ...(route === "personal-key" || route === "oauth"
+ ? { providerKeys: { anthropic: route === "oauth" ? "sk-ant-oat-offline-test" : "sk-personal-test" } }
+ : {}),
+ emit: async (entry) => ({ ...entry, seq: 1, createdAt: Date.now() }) as never,
+ recordModelCall: () => {},
+ cancel: AbortSignal.timeout(10_000),
+ });
+ assert.equal(result.reply, "OK");
+ const request = requests.at(-1)!;
+ const assistant = request.body.messages.find((message: { role: string }) => message.role === "assistant");
+ assert.deepEqual(
+ assistant.content.find((block: { type: string }) => block.type === "thinking"),
+ { type: "thinking", thinking: "Use the attachment tool.", signature: "test-preserved-signature" },
+ );
+ assert.equal(request.body.temperature, undefined);
+ assert.equal(request.body.thinking?.budget_tokens, undefined);
+ assert.equal(request.body.max_tokens, getRequiredModel(modelId).maxTokens);
+ if (effortLevel === "adaptive" || effortLevel === "default")
+ assert.equal(request.body.output_config?.effort, undefined);
+ else assert.equal(request.body.output_config?.effort, effortLevel === "auto" ? "low" : effortLevel);
+ if (
+ effortLevel === "default" &&
+ (route === "gateway" || getRequiredModel(modelId).thinkingLevelMap?.off !== null)
+ )
+ assert.equal(request.body.thinking, undefined);
+ else assert.equal(request.body.thinking?.type, "adaptive");
+ if (route === "gateway") {
+ assert.equal(request.body.model, "routed-model");
+ assert.equal(request.headers.get("x-gateway-key"), "test-key");
+ }
+ if (route === "oauth") assert.match(request.headers.get("authorization") ?? "", /^Bearer /);
+ if (route === "personal-key") assert.equal(request.headers.get("x-api-key"), "sk-personal-test");
+ }
+ });
+ }
+});
+
+test("runtime scope writes and turn dispatch reject unsupported modes without changing legacy selections", async () => {
+ const { createMemoryConfigStore } = await import("../src/resolution/config-store.ts");
+ const { resolveRuntimeChoice } = await import("../src/harness/harness-router.ts");
+ const { createRuntimeService } = await import("../src/harness/runtime-control.ts");
+ const config = createMemoryConfigStore("default-org");
+ config.setApprovedHarnesses(["pi", "claude", "codex", "opencode"]);
+ const active = { harnessId: "pi" as const, modelId: "claude-sonnet-5", effortLevel: "auto" };
+ await config.setRuntimeSelectionLatest("personal:test", active);
+ const service = createRuntimeService({ config, harnessId: "pi" }, { authorizesCapabilityScope: async () => true });
+ const claims = { actorId: "test", scopeId: "personal:test" as const, liveActor: true, exp: Date.now() + 60_000 };
+ for (const effort of ["adaptive", "default"]) {
+ const result = await service(claims, active, { action: "set", effort, lifetime: "scope" });
+ assert.equal(result.ok, true);
+ assert.equal(resolveRuntimeChoice(config, "org:default-org", "personal:test", active).effortLevel, effort);
+ }
+ assert.deepEqual(
+ await service(claims, active, { action: "set", effort: "adaptive", model: "gpt-6-astra", lifetime: "scope" }),
+ { ok: false, error: "effort_not_supported" },
+ );
+ assert.equal(config.getRuntimeSelection("personal:test")?.effortLevel, "default");
+ assert.throws(
+ () =>
+ resolveRuntimeChoice(config, "org:default-org", "personal:test", active, {
+ modelId: "gpt-6-astra",
+ effortLevel: "adaptive",
+ }),
+ /not supported/,
+ );
+ await config.setRuntimeSelectionLatest("personal:test", active);
+ assert.equal(resolveRuntimeChoice(config, "org:default-org", "personal:test", active).effortLevel, "auto");
+});
diff --git a/test/pi-harness-effort-wire.test.ts b/test/pi-harness-effort-wire.test.ts
index 2d8fb1625..b55d383cc 100644
--- a/test/pi-harness-effort-wire.test.ts
+++ b/test/pi-harness-effort-wire.test.ts
@@ -11,18 +11,65 @@ for (const [modelId, upper, allowed] of [
["gpt-5", "high", ["minimal", "low", "medium", "high"]],
] as const) {
test(`Pi sends supported reasoning through HTTP for ${modelId}`, async (t) => {
- const bodies: Array<{ reasoning?: { effort?: string } }> = [];
+ const bodies: Array<{
+ reasoning?: { effort?: string };
+ include?: string[];
+ input: Array<Record<string, unknown>>;
+ }> = [];
+ let sendTool = false;
const server = createServer(async (req, res) => {
let raw = "";
for await (const chunk of req) raw += String(chunk);
const body = JSON.parse(raw);
bodies.push(body);
- if (!(allowed as readonly string[]).includes(body.reasoning?.effort)) {
+ if (body.reasoning && !(allowed as readonly string[]).includes(body.reasoning?.effort)) {
res.writeHead(400, { "content-type": "application/json" });
res.end(JSON.stringify({ error: { message: "Unsupported reasoning.effort", type: "invalid_request_error" } }));
return;
}
res.writeHead(200, { "content-type": "text/event-stream" });
+ if (sendTool) {
+ sendTool = false;
+ const reasoning = {
+ id: "rs_native",
+ type: "reasoning",
+ summary: [],
+ ...(body.include?.includes("reasoning.encrypted_content")
+ ? { encrypted_content: "encrypted-native-reasoning" }
+ : {}),
+ };
+ const call = {
+ id: "fc_native",
+ type: "function_call",
+ call_id: "call_native",
+ name: "attach",
+ arguments: JSON.stringify({ files: ["probe.txt"] }),
+ status: "completed",
+ };
+ for (const event of [
+ {
+ type: "response.created",
+ response: { id: "resp_tool", model: modelId, status: "in_progress", output: [] },
+ },
+ { type: "response.output_item.added", output_index: 0, item: reasoning },
+ { type: "response.output_item.done", output_index: 0, item: reasoning },
+ { type: "response.output_item.added", output_index: 1, item: { ...call, arguments: "" } },
+ { type: "response.output_item.done", output_index: 1, item: call },
+ {
+ type: "response.completed",
+ response: {
+ id: "resp_tool",
+ model: modelId,
+ status: "completed",
+ output: [reasoning, call],
+ usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
+ },
+ },
+ ])
+ res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
+ res.end();
+ return;
+ }
const message = {
id: "msg_test",
type: "message",
@@ -81,15 +128,17 @@ for (const [modelId, upper, allowed] of [
["off", modelId === "gpt-5" ? "minimal" : "none"],
["xhigh", upper],
["auto", "medium"],
+ ["default", undefined],
]) {
await t.test(effortLevel!, async () => {
let seq = 0;
+ sendTool = effortLevel === "default";
const result = await harness.turns.runTurn({
session: { id: `effort-${modelId}` } as HarnessTurnInput["session"],
input: "hello",
systemPrompt: "Reply ok.",
history: [],
- tools: {} as HarnessTurnInput["tools"],
+ tools: { attach: async () => ({ ok: true, files: [], staged: 0 }) } as unknown as HarnessTurnInput["tools"],
scopeLabel: "personal:test",
orgScopeId: "org:test",
runtime: { effortLevel },
@@ -99,6 +148,14 @@ for (const [modelId, upper, allowed] of [
});
assert.equal(result.reply, "ok");
assert.equal(bodies.at(-1)?.reasoning?.effort, expected);
+ if (effortLevel === "default") {
+ assert.ok(bodies.at(-1)?.include?.includes("reasoning.encrypted_content"));
+ assert.equal(
+ bodies.at(-1)?.input.find((item) => item.type === "reasoning")?.encrypted_content,
+ "encrypted-native-reasoning",
+ );
+ assert.ok(bodies.at(-1)?.input.some((item) => item.type === "function_call_output"));
+ }
});
}
assert.deepEqual(