Multiplayer agent harness for work.
Stop a mock deployment passing as a real one (#25)
Joshua France3300ec42ece5
diff --git a/cli/src/backends/doctor.ts b/cli/src/backends/doctor.tsindex 837139193..7abf0ad23 100644--- a/cli/src/backends/doctor.ts+++ b/cli/src/backends/doctor.ts@@ -1,8 +1,8 @@import { existsSync, readFileSync } from "node:fs";import { join, resolve } from "node:path";import {- MODEL_PROVIDER_BASE_MODELS,MODEL_PROVIDER_KEYS,+ mockHarnessWarning,validatePortalTrust,type ModelProvider,type QmConfig,@@ -199,6 +199,8 @@ export async function doctorCommon(}async function baseModelCheck(config: QmConfig, secrets: Map<string, string>): Promise<void> {+ const mockHarness = mockHarnessWarning(config);+ if (mockHarness) warn(mockHarness);const provider = config.modelProvider;if (!provider) {step("base model: no modelProvider set — an administrator supplies the key from the Admin page");@@ -211,7 +213,7 @@ async function baseModelCheck(config: QmConfig, secrets: Map<string, string>): Preturn;}await modelProviderCheck(provider, key);- step(`base model provider ${provider}: ${name} accepted, serving ${MODEL_PROVIDER_BASE_MODELS[provider]}`);+ step(`base model provider ${provider}: ${name} accepted`);}const MODEL_PROVIDER_PROBES: Readonly<diff --git a/cli/src/commands/check.ts b/cli/src/commands/check.tsindex f8ec83196..fa258e337 100644--- a/cli/src/commands/check.ts+++ b/cli/src/commands/check.ts@@ -3,7 +3,7 @@ import { relative, resolve } from "node:path";import { CliError, errMessage, header, note, ok, step, warn } from "../log.ts";import { validateSandboxLayer, type SandboxValidation } from "../sandbox-layer.ts";import { discoverPlugins, type ResolvedPlugin } from "../plugins.ts";-import { sandboxPinPending, type QmConfig } from "../config.ts";+import { mockHarnessWarning, sandboxPinPending, type QmConfig } from "../config.ts";import { computedSecrets, runtimeSecretNames, type ComputedSecret } from "../secrets.ts";import { isVirtualService, runnableServices } from "../services.ts";import { serviceEnvironment } from "../backends/aws.ts";@@ -143,6 +143,8 @@ export function runChecks(const optional = secrets.filter((secret) => !secret.required).map((secret) => secret.name);if (optional.length) step(`optional secrets: ${optional.join(", ")}`);for (const w of layer.warnings) warn(w);+ const mockHarness = mockHarnessWarning(config);+ if (mockHarness) warn(mockHarness);if (sandboxPinPending(config)) {warn("no sandbox layer image is pinned yet; run `qm sandbox publish` to record one before `qm up` renders core");}diff --git a/cli/src/config.ts b/cli/src/config.tsindex 9c0be6671..4d1f283ea 100644--- a/cli/src/config.ts+++ b/cli/src/config.ts@@ -119,12 +119,6 @@ export const MODEL_PROVIDER_HARNESSES: Readonly<Record<ModelProvider, readonly sopenrouter: ["pi", "mock"],};-export const MODEL_PROVIDER_BASE_MODELS: Readonly<Record<ModelProvider, string>> = {- anthropic: "claude-opus-5",- openai: "gpt-5.6-sol",- openrouter: "openrouter/auto",-};-export const isModelProvider = (value: unknown): value is ModelProvider =>typeof value === "string" && (MODEL_PROVIDERS as readonly string[]).includes(value);@@ -714,6 +708,12 @@ function configuredHarness(config: QmConfig): string {return config.env.core?.HARNESS?.trim() || (config.target === "fly" ? "pi" : "mock");}+export function mockHarnessWarning(config: QmConfig): string | undefined {+ if (configuredHarness(config) !== "mock") return undefined;+ const unset = !config.env.core?.HARNESS?.trim();+ return `env.core.HARNESS is ${unset ? "unset, which means" : "set to"} "mock": this deployment answers every message with canned text and calls no model provider. Set it to "pi" for a deployment that runs real agent turns.`;+}+function validateModelProvider(config: QmConfig, path: string): void {const override = config.env.core?.MODEL_PROVIDER?.trim();if (override !== undefined && override !== "" && !isModelProvider(override)) {diff --git a/cli/test/config.test.ts b/cli/test/config.test.tsindex bd30c0441..7d62298de 100644--- a/cli/test/config.test.ts+++ b/cli/test/config.test.ts@@ -7,6 +7,7 @@ import {CONFIG_FILENAME,loadConfigAt,loadConfigInDir,+ mockHarnessWarning,sandboxCoreEnv,sandboxImagePinErrors,sandboxPinPending,@@ -1178,3 +1179,18 @@ test("env.core.MODEL_PROVIDER is validated as the provider core will actually usassert.throws(() => loadConfigAt(path), /env.core.MODEL_PROVIDER must be one of/);});});++test("a mock deployment is named as one, and a real harness draws no warning", () => {+ withConfig({ env: { core: {} } }, ({ path }) => {+ assert.match(mockHarnessWarning(loadConfigAt(path).config)!, /unset, which means "mock".*calls no model provider/s);+ });+ withConfig({ env: { core: { HARNESS: "mock" } } }, ({ path }) => {+ assert.match(mockHarnessWarning(loadConfigAt(path).config)!, /set to "mock"/);+ });+ withConfig({ env: { core: { HARNESS: "pi" } } }, ({ path }) => {+ assert.equal(mockHarnessWarning(loadConfigAt(path).config), undefined);+ });+ withConfig({ target: "fly", appPrefix: "acme", env: { core: {} } }, ({ path }) => {+ assert.equal(mockHarnessWarning(loadConfigAt(path).config), undefined, "the fly template renders HARNESS=pi");+ });+});diff --git a/src/config.ts b/src/config.tsindex c274c4e7e..534134b09 100644--- a/src/config.ts+++ b/src/config.ts@@ -565,6 +565,11 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {);}}+ if (env.NODE_ENV === "production" && harnessEnvStrict(env.HARNESS) === "mock") {+ console.warn(+ `[config] HARNESS is ${env.HARNESS?.trim() ? '"mock"' : "unset, which means mock"} in production — this deployment answers every message with canned text and calls no model provider. Set HARNESS=pi to run real agent turns.`,+ );+ }if (env.SANDBOX_BACKEND === "sprites" && !env.SPRITES_EGRESS_PROXY_URL) {console.warn("[config] SANDBOX_BACKEND=sprites without SPRITES_EGRESS_PROXY_URL — sandboxes run with NO egress enforcement (fail-open); set SPRITES_EGRESS_PROXY_URL to the public egress proxy to force sandbox traffic through it.",diff --git a/test/config.test.ts b/test/config.test.tsindex 7d89b29ba..ee5ba307f 100644--- a/test/config.test.ts+++ b/test/config.test.ts@@ -136,6 +136,24 @@ test("harness security posture defaults to auto and validates named modes", () =);});+test("production names a mock harness rather than letting it pass as a real deployment", () => {+ const warnings: string[] = [];+ const original = console.warn;+ console.warn = (msg: unknown) => void warnings.push(String(msg));+ try {+ loadConfig(productionEnv);+ loadConfig({ ...productionEnv, HARNESS: "mock" });+ loadConfig({ ...productionEnv, HARNESS: "pi" });+ loadConfig({});+ } finally {+ console.warn = original;+ }+ const mock = warnings.filter((w) => w.includes("calls no model provider"));+ assert.equal(mock.length, 2, "production + unset and production + mock each warn once");+ assert.match(mock[0]!, /unset, which means mock/);+ assert.match(mock[1]!, /HARNESS is "mock"/);+});+test("a leftover *=sqlite env throws (no silent downgrade to ephemeral memory)", () => {assert.throws(() => loadConfig({ SESSION_STORE: "sqlite" }), /SESSION_STORE=sqlite is no longer supported/);assert.throws(() => loadConfig({ RUN_STORE: "sqlite" }), /RUN_STORE=sqlite is no longer supported/);diff --git a/test/secret-schema-drift.test.ts b/test/secret-schema-drift.test.tsindex 209eaf1ac..cb71f8899 100644--- a/test/secret-schema-drift.test.ts+++ b/test/secret-schema-drift.test.ts@@ -2,7 +2,7 @@ import test from "node:test";import assert from "node:assert/strict";import { readFileSync } from "node:fs";import { FIRST_PARTY_SECRET_SPECS } from "../cli/src/secrets.ts";-import { MODEL_PROVIDERS, MODEL_PROVIDER_BASE_MODELS, MODEL_PROVIDER_HARNESSES } from "../cli/src/config.ts";+import { MODEL_PROVIDERS, MODEL_PROVIDER_HARNESSES } from "../cli/src/config.ts";import { CORE_SECRET_SPECS, validateCoreSecretEnv } from "../src/deployment/secret-schema.ts";import { HARNESS_IDS, defaultModelForProvider } from "../src/model/pi-models.ts";@@ -83,18 +83,6 @@ test("the CLI's provider/harness table matches what the core model registry can}});-test("the CLI names the same base model the runtime will pick for each provider", () => {- for (const provider of MODEL_PROVIDERS) {- for (const harness of MODEL_PROVIDER_HARNESSES[provider]) {- assert.equal(- defaultModelForProvider(harness, provider),- MODEL_PROVIDER_BASE_MODELS[provider],- `MODEL_PROVIDER_BASE_MODELS.${provider} has drifted for HARNESS=${harness}`,- );- }- }-});-test("production rejects weak encryption key material for managed credentials", () => {const strong = "x".repeat(32);const env = {