Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Stop a mock deployment passing as a real one (#25)

Joshua France3300ec42ece5
diff --git a/cli/src/backends/doctor.ts b/cli/src/backends/doctor.ts
index 837139193..7abf0ad23 100644
--- a/cli/src/backends/doctor.ts
+++ b/cli/src/backends/doctor.ts
@@ -1,8 +1,8 @@
import { existsSync, readFileSync } from "node:fs";
import { join, resolve } from "node:path";
import {
- MODEL_PROVIDER_BASE_MODELS,
MODEL_PROVIDER_KEYS,
+ mockHarnessWarning,
validatePortalTrust,
type ModelProvider,
type QmConfig,
@@ -199,6 +199,8 @@ export async function doctorCommon(
}
async function baseModelCheck(config: QmConfig, secrets: Map<string, string>): Promise<void> {
+ const mockHarness = mockHarnessWarning(config);
+ if (mockHarness) warn(mockHarness);
const provider = config.modelProvider;
if (!provider) {
step("base model: no modelProvider set — an administrator supplies the key from the Admin page");
@@ -211,7 +213,7 @@ async function baseModelCheck(config: QmConfig, secrets: Map<string, string>): P
return;
}
await modelProviderCheck(provider, key);
- step(`base model provider ${provider}: ${name} accepted, serving ${MODEL_PROVIDER_BASE_MODELS[provider]}`);
+ step(`base model provider ${provider}: ${name} accepted`);
}
const MODEL_PROVIDER_PROBES: Readonly<
diff --git a/cli/src/commands/check.ts b/cli/src/commands/check.ts
index f8ec83196..fa258e337 100644
--- a/cli/src/commands/check.ts
+++ b/cli/src/commands/check.ts
@@ -3,7 +3,7 @@ import { relative, resolve } from "node:path";
import { CliError, errMessage, header, note, ok, step, warn } from "../log.ts";
import { validateSandboxLayer, type SandboxValidation } from "../sandbox-layer.ts";
import { discoverPlugins, type ResolvedPlugin } from "../plugins.ts";
-import { sandboxPinPending, type QmConfig } from "../config.ts";
+import { mockHarnessWarning, sandboxPinPending, type QmConfig } from "../config.ts";
import { computedSecrets, runtimeSecretNames, type ComputedSecret } from "../secrets.ts";
import { isVirtualService, runnableServices } from "../services.ts";
import { serviceEnvironment } from "../backends/aws.ts";
@@ -143,6 +143,8 @@ export function runChecks(
const optional = secrets.filter((secret) => !secret.required).map((secret) => secret.name);
if (optional.length) step(`optional secrets: ${optional.join(", ")}`);
for (const w of layer.warnings) warn(w);
+ const mockHarness = mockHarnessWarning(config);
+ if (mockHarness) warn(mockHarness);
if (sandboxPinPending(config)) {
warn("no sandbox layer image is pinned yet; run `qm sandbox publish` to record one before `qm up` renders core");
}
diff --git a/cli/src/config.ts b/cli/src/config.ts
index 9c0be6671..4d1f283ea 100644
--- a/cli/src/config.ts
+++ b/cli/src/config.ts
@@ -119,12 +119,6 @@ export const MODEL_PROVIDER_HARNESSES: Readonly<Record<ModelProvider, readonly s
openrouter: ["pi", "mock"],
};
-export const MODEL_PROVIDER_BASE_MODELS: Readonly<Record<ModelProvider, string>> = {
- anthropic: "claude-opus-5",
- openai: "gpt-5.6-sol",
- openrouter: "openrouter/auto",
-};
-
export const isModelProvider = (value: unknown): value is ModelProvider =>
typeof value === "string" && (MODEL_PROVIDERS as readonly string[]).includes(value);
@@ -714,6 +708,12 @@ function configuredHarness(config: QmConfig): string {
return config.env.core?.HARNESS?.trim() || (config.target === "fly" ? "pi" : "mock");
}
+export function mockHarnessWarning(config: QmConfig): string | undefined {
+ if (configuredHarness(config) !== "mock") return undefined;
+ const unset = !config.env.core?.HARNESS?.trim();
+ return `env.core.HARNESS is ${unset ? "unset, which means" : "set to"} "mock": this deployment answers every message with canned text and calls no model provider. Set it to "pi" for a deployment that runs real agent turns.`;
+}
+
function validateModelProvider(config: QmConfig, path: string): void {
const override = config.env.core?.MODEL_PROVIDER?.trim();
if (override !== undefined && override !== "" && !isModelProvider(override)) {
diff --git a/cli/test/config.test.ts b/cli/test/config.test.ts
index bd30c0441..7d62298de 100644
--- a/cli/test/config.test.ts
+++ b/cli/test/config.test.ts
@@ -7,6 +7,7 @@ import {
CONFIG_FILENAME,
loadConfigAt,
loadConfigInDir,
+ mockHarnessWarning,
sandboxCoreEnv,
sandboxImagePinErrors,
sandboxPinPending,
@@ -1178,3 +1179,18 @@ test("env.core.MODEL_PROVIDER is validated as the provider core will actually us
assert.throws(() => loadConfigAt(path), /env.core.MODEL_PROVIDER must be one of/);
});
});
+
+test("a mock deployment is named as one, and a real harness draws no warning", () => {
+ withConfig({ env: { core: {} } }, ({ path }) => {
+ assert.match(mockHarnessWarning(loadConfigAt(path).config)!, /unset, which means "mock".*calls no model provider/s);
+ });
+ withConfig({ env: { core: { HARNESS: "mock" } } }, ({ path }) => {
+ assert.match(mockHarnessWarning(loadConfigAt(path).config)!, /set to "mock"/);
+ });
+ withConfig({ env: { core: { HARNESS: "pi" } } }, ({ path }) => {
+ assert.equal(mockHarnessWarning(loadConfigAt(path).config), undefined);
+ });
+ withConfig({ target: "fly", appPrefix: "acme", env: { core: {} } }, ({ path }) => {
+ assert.equal(mockHarnessWarning(loadConfigAt(path).config), undefined, "the fly template renders HARNESS=pi");
+ });
+});
diff --git a/src/config.ts b/src/config.ts
index c274c4e7e..534134b09 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -565,6 +565,11 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {
);
}
}
+ if (env.NODE_ENV === "production" && harnessEnvStrict(env.HARNESS) === "mock") {
+ console.warn(
+ `[config] HARNESS is ${env.HARNESS?.trim() ? '"mock"' : "unset, which means mock"} in production — this deployment answers every message with canned text and calls no model provider. Set HARNESS=pi to run real agent turns.`,
+ );
+ }
if (env.SANDBOX_BACKEND === "sprites" && !env.SPRITES_EGRESS_PROXY_URL) {
console.warn(
"[config] SANDBOX_BACKEND=sprites without SPRITES_EGRESS_PROXY_URL — sandboxes run with NO egress enforcement (fail-open); set SPRITES_EGRESS_PROXY_URL to the public egress proxy to force sandbox traffic through it.",
diff --git a/test/config.test.ts b/test/config.test.ts
index 7d89b29ba..ee5ba307f 100644
--- a/test/config.test.ts
+++ b/test/config.test.ts
@@ -136,6 +136,24 @@ test("harness security posture defaults to auto and validates named modes", () =
);
});
+test("production names a mock harness rather than letting it pass as a real deployment", () => {
+ const warnings: string[] = [];
+ const original = console.warn;
+ console.warn = (msg: unknown) => void warnings.push(String(msg));
+ try {
+ loadConfig(productionEnv);
+ loadConfig({ ...productionEnv, HARNESS: "mock" });
+ loadConfig({ ...productionEnv, HARNESS: "pi" });
+ loadConfig({});
+ } finally {
+ console.warn = original;
+ }
+ const mock = warnings.filter((w) => w.includes("calls no model provider"));
+ assert.equal(mock.length, 2, "production + unset and production + mock each warn once");
+ assert.match(mock[0]!, /unset, which means mock/);
+ assert.match(mock[1]!, /HARNESS is "mock"/);
+});
+
test("a leftover *=sqlite env throws (no silent downgrade to ephemeral memory)", () => {
assert.throws(() => loadConfig({ SESSION_STORE: "sqlite" }), /SESSION_STORE=sqlite is no longer supported/);
assert.throws(() => loadConfig({ RUN_STORE: "sqlite" }), /RUN_STORE=sqlite is no longer supported/);
diff --git a/test/secret-schema-drift.test.ts b/test/secret-schema-drift.test.ts
index 209eaf1ac..cb71f8899 100644
--- a/test/secret-schema-drift.test.ts
+++ b/test/secret-schema-drift.test.ts
@@ -2,7 +2,7 @@ import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { FIRST_PARTY_SECRET_SPECS } from "../cli/src/secrets.ts";
-import { MODEL_PROVIDERS, MODEL_PROVIDER_BASE_MODELS, MODEL_PROVIDER_HARNESSES } from "../cli/src/config.ts";
+import { MODEL_PROVIDERS, MODEL_PROVIDER_HARNESSES } from "../cli/src/config.ts";
import { CORE_SECRET_SPECS, validateCoreSecretEnv } from "../src/deployment/secret-schema.ts";
import { HARNESS_IDS, defaultModelForProvider } from "../src/model/pi-models.ts";
@@ -83,18 +83,6 @@ test("the CLI's provider/harness table matches what the core model registry can
}
});
-test("the CLI names the same base model the runtime will pick for each provider", () => {
- for (const provider of MODEL_PROVIDERS) {
- for (const harness of MODEL_PROVIDER_HARNESSES[provider]) {
- assert.equal(
- defaultModelForProvider(harness, provider),
- MODEL_PROVIDER_BASE_MODELS[provider],
- `MODEL_PROVIDER_BASE_MODELS.${provider} has drifted for HARNESS=${harness}`,
- );
- }
- }
-});
-
test("production rejects weak encryption key material for managed credentials", () => {
const strong = "x".repeat(32);
const env = {