Multiplayer agent harness for work.
fix: strengthen model verification fingerprints
Josh France2c8bd8a44b21
diff --git a/src/model/model-verification.ts b/src/model/model-verification.tsindex 7078648e5..dfe2e0a6e 100644--- a/src/model/model-verification.ts+++ b/src/model/model-verification.ts@@ -1,4 +1,4 @@-import { createHmac } from "node:crypto";+import { createHmac, scrypt } from "node:crypto";import { probeModel } from "../harness/pi-harness.ts";import { modelFromOverlay } from "./pi-models.ts";import type { ModelOverlay } from "./model-overlay.ts";@@ -61,6 +61,26 @@ export function createModelVerifier(input: {modelGateway?: ModelGatewayTransportConfig;probe?: typeof probeModel;}): ModelVerifier {+ const salt = createHmac("sha256", input.keyMaterial).update("qm:model-verification:credential:v2").digest();+ const credentialKeys = new Map<+ ModelOverlay["provider"] | "gateway",+ { material: string; derived: Promise<Buffer> }+ >();+ function credentialKey(source: ModelOverlay["provider"] | "gateway", material: string): Promise<Buffer> {+ const cached = credentialKeys.get(source);+ if (cached?.material === material) return cached.derived;+ const derived = new Promise<Buffer>((resolve, reject) => {+ scrypt(material, salt, 32, { N: 131_072, r: 8, p: 1, maxmem: 256 * 1024 * 1024 }, (error, key) => {+ if (error) reject(error);+ else resolve(key);+ });+ }).catch((error: unknown) => {+ if (credentialKeys.get(source)?.derived === derived) credentialKeys.delete(source);+ throw error;+ });+ credentialKeys.set(source, { material, derived });+ return derived;+ }return async (spec) => {const model = modelFromOverlay(spec);if (!model)@@ -76,17 +96,18 @@ export function createModelVerifier(input: {"Configure an organization provider credential before verifying this model.",);const status = (await input.credentials.statuses()).find((s) => s.provider === spec.provider);- const fingerprint = createHmac("sha256", input.keyMaterial)+ const derived = await credentialKey(+ gateway ? "gateway" : spec.provider,+ JSON.stringify({ key, apiKeyHeader: gateway?.apiKeyHeader }),+ );+ const fingerprint = createHmac("sha256", derived).update(JSON.stringify({- version: 1,+ version: 2,spec,model,- key,credentialRevision: gateway ? undefined : status,- gateway: gateway- ? { url: gateway.url, apiKeyHeader: gateway.apiKeyHeader, target: gateway.models[spec.id] }- : undefined,+ gateway: gateway ? { url: gateway.url, target: gateway.models[spec.id] } : undefined,}),).digest("hex");diff --git a/test/model-verification.test.ts b/test/model-verification.test.tsindex 3d3ec9628..7a1988f0b 100644--- a/test/model-verification.test.ts+++ b/test/model-verification.test.ts@@ -431,3 +431,109 @@ test("the store deadline leaves a hanging provider unverified", { timeout: 25_00setProviderBaseUrls({});}});++import { createHmac, scryptSync } from "node:crypto";++test("verification fingerprints use a memory-hard credential-bound MAC key", async () => {+ const built = buildApp(testConfig({ openaiApiKey: "fingerprint-test-key" }));+ const keyMaterial = "deployment-fingerprint-test-key";+ const candidate = parseModelOverlay(spec);+ const verifier = createModelVerifier({ credentials: built.modelCredentials, keyMaterial });+ const salt = createHmac("sha256", keyMaterial).update("qm:model-verification:credential:v2").digest();+ const derived = scryptSync(JSON.stringify({ key: "fingerprint-test-key" }), salt, 32, {+ N: 131_072,+ r: 8,+ p: 1,+ maxmem: 256 * 1024 * 1024,+ });+ const expected = createHmac("sha256", derived)+ .update(+ JSON.stringify({+ version: 2,+ spec: candidate,+ model: modelFromOverlay(candidate),+ credentialRevision: (await built.modelCredentials.statuses()).find(+ (status) => status.provider === candidate.provider,+ ),+ }),+ )+ .digest("hex");+ assert.equal((await verifier(candidate)).fingerprint, expected);+ const concurrent = await Promise.all(Array.from({ length: 8 }, () => verifier(candidate)));+ assert.ok(concurrent.every((value) => value.fingerprint === expected));+ const restarted = createModelVerifier({ credentials: built.modelCredentials, keyMaterial });+ assert.equal((await restarted(candidate)).fingerprint, expected);+ const differentDeployment = createModelVerifier({+ credentials: built.modelCredentials,+ keyMaterial: "other-deployment-key",+ });+ assert.notEqual((await differentDeployment(candidate)).fingerprint, expected);+});++test("environment and gateway credential changes invalidate cached fingerprints", async () => {+ const fallback = { openai: "first-environment-test-key" };+ const credentials = createModelCredentialStore({ backing: createMemoryMap(), keyMaterial: "test-key", fallback });+ const candidate = parseModelOverlay(spec);+ const modelGateway = {+ url: "https://gateway.example.invalid/v1",+ apiKey: "first-gateway-test-key",+ apiKeyHeader: "x-gateway-key",+ models: { [spec.id]: "target" },+ };+ const direct = createModelVerifier({ credentials, keyMaterial: "test-key" });+ const gateway = createModelVerifier({ credentials, keyMaterial: "test-key", modelGateway });+ const directOriginal = (await direct(candidate)).fingerprint;+ const gatewayOriginal = (await gateway(candidate)).fingerprint;+ fallback.openai = "changed-environment-test-key";+ assert.notEqual((await direct(candidate)).fingerprint, directOriginal);+ assert.equal((await gateway(candidate)).fingerprint, gatewayOriginal);+ modelGateway.apiKey = "changed-gateway-test-key";+ const rotatedGateway = (await gateway(candidate)).fingerprint;+ assert.notEqual(rotatedGateway, gatewayOriginal);+ modelGateway.apiKeyHeader = "x-new-gateway-key";+ assert.notEqual((await gateway(candidate)).fingerprint, rotatedGateway);+});++import { createModelCredentialStore } from "../src/model/model-credential-store.ts";++test("legacy fingerprint versions stay unavailable until a successful new verification", async () => {+ const built = buildApp(testConfig({ openaiApiKey: "test-key" }));+ const candidate = parseModelOverlay(spec);+ const keyMaterial = "stable-test-key";+ const fingerprint = createHmac("sha256", keyMaterial)+ .update(+ JSON.stringify({+ version: 1,+ spec: candidate,+ model: modelFromOverlay(candidate),+ key: "test-key",+ credentialRevision: (await built.modelCredentials.statuses()).find((status) => status.provider === "openai"),+ }),+ )+ .digest("hex");+ const backing = createMemoryMap<StoredModelOverlay>();+ await backing.put(spec.id, {+ spec: candidate,+ disabled: false,+ updatedAt: 1,+ updatedBy: "admin",+ verification: { fingerprint, verifiedAt: 1, revision: "old-proof" },+ });+ let probes = 0;+ const verifier = createModelVerifier({+ credentials: built.modelCredentials,+ keyMaterial,+ probe: async () => {+ probes++;+ },+ });+ const store = createModelOverlayStore(backing, undefined, verifier);+ await store.refresh();+ assert.equal(probes, 0);+ assert.equal(resolveModel(spec.id), undefined);+ assert.match(modelUnavailableReason(spec.id)!, /verify.*again/i);+ await store.upsert(candidate, "admin");+ await store.refresh();+ assert.equal(probes, 1);+ assert.ok(resolveModel(spec.id));+});