Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

fix: strengthen model verification fingerprints

Josh France2c8bd8a44b21
diff --git a/src/model/model-verification.ts b/src/model/model-verification.ts
index 7078648e5..dfe2e0a6e 100644
--- a/src/model/model-verification.ts
+++ b/src/model/model-verification.ts
@@ -1,4 +1,4 @@
-import { createHmac } from "node:crypto";
+import { createHmac, scrypt } from "node:crypto";
import { probeModel } from "../harness/pi-harness.ts";
import { modelFromOverlay } from "./pi-models.ts";
import type { ModelOverlay } from "./model-overlay.ts";
@@ -61,6 +61,26 @@ export function createModelVerifier(input: {
modelGateway?: ModelGatewayTransportConfig;
probe?: typeof probeModel;
}): ModelVerifier {
+ const salt = createHmac("sha256", input.keyMaterial).update("qm:model-verification:credential:v2").digest();
+ const credentialKeys = new Map<
+ ModelOverlay["provider"] | "gateway",
+ { material: string; derived: Promise<Buffer> }
+ >();
+ function credentialKey(source: ModelOverlay["provider"] | "gateway", material: string): Promise<Buffer> {
+ const cached = credentialKeys.get(source);
+ if (cached?.material === material) return cached.derived;
+ const derived = new Promise<Buffer>((resolve, reject) => {
+ scrypt(material, salt, 32, { N: 131_072, r: 8, p: 1, maxmem: 256 * 1024 * 1024 }, (error, key) => {
+ if (error) reject(error);
+ else resolve(key);
+ });
+ }).catch((error: unknown) => {
+ if (credentialKeys.get(source)?.derived === derived) credentialKeys.delete(source);
+ throw error;
+ });
+ credentialKeys.set(source, { material, derived });
+ return derived;
+ }
return async (spec) => {
const model = modelFromOverlay(spec);
if (!model)
@@ -76,17 +96,18 @@ export function createModelVerifier(input: {
"Configure an organization provider credential before verifying this model.",
);
const status = (await input.credentials.statuses()).find((s) => s.provider === spec.provider);
- const fingerprint = createHmac("sha256", input.keyMaterial)
+ const derived = await credentialKey(
+ gateway ? "gateway" : spec.provider,
+ JSON.stringify({ key, apiKeyHeader: gateway?.apiKeyHeader }),
+ );
+ const fingerprint = createHmac("sha256", derived)
.update(
JSON.stringify({
- version: 1,
+ version: 2,
spec,
model,
- key,
credentialRevision: gateway ? undefined : status,
- gateway: gateway
- ? { url: gateway.url, apiKeyHeader: gateway.apiKeyHeader, target: gateway.models[spec.id] }
- : undefined,
+ gateway: gateway ? { url: gateway.url, target: gateway.models[spec.id] } : undefined,
}),
)
.digest("hex");
diff --git a/test/model-verification.test.ts b/test/model-verification.test.ts
index 3d3ec9628..7a1988f0b 100644
--- a/test/model-verification.test.ts
+++ b/test/model-verification.test.ts
@@ -431,3 +431,109 @@ test("the store deadline leaves a hanging provider unverified", { timeout: 25_00
setProviderBaseUrls({});
}
});
+
+import { createHmac, scryptSync } from "node:crypto";
+
+test("verification fingerprints use a memory-hard credential-bound MAC key", async () => {
+ const built = buildApp(testConfig({ openaiApiKey: "fingerprint-test-key" }));
+ const keyMaterial = "deployment-fingerprint-test-key";
+ const candidate = parseModelOverlay(spec);
+ const verifier = createModelVerifier({ credentials: built.modelCredentials, keyMaterial });
+ const salt = createHmac("sha256", keyMaterial).update("qm:model-verification:credential:v2").digest();
+ const derived = scryptSync(JSON.stringify({ key: "fingerprint-test-key" }), salt, 32, {
+ N: 131_072,
+ r: 8,
+ p: 1,
+ maxmem: 256 * 1024 * 1024,
+ });
+ const expected = createHmac("sha256", derived)
+ .update(
+ JSON.stringify({
+ version: 2,
+ spec: candidate,
+ model: modelFromOverlay(candidate),
+ credentialRevision: (await built.modelCredentials.statuses()).find(
+ (status) => status.provider === candidate.provider,
+ ),
+ }),
+ )
+ .digest("hex");
+ assert.equal((await verifier(candidate)).fingerprint, expected);
+ const concurrent = await Promise.all(Array.from({ length: 8 }, () => verifier(candidate)));
+ assert.ok(concurrent.every((value) => value.fingerprint === expected));
+ const restarted = createModelVerifier({ credentials: built.modelCredentials, keyMaterial });
+ assert.equal((await restarted(candidate)).fingerprint, expected);
+ const differentDeployment = createModelVerifier({
+ credentials: built.modelCredentials,
+ keyMaterial: "other-deployment-key",
+ });
+ assert.notEqual((await differentDeployment(candidate)).fingerprint, expected);
+});
+
+test("environment and gateway credential changes invalidate cached fingerprints", async () => {
+ const fallback = { openai: "first-environment-test-key" };
+ const credentials = createModelCredentialStore({ backing: createMemoryMap(), keyMaterial: "test-key", fallback });
+ const candidate = parseModelOverlay(spec);
+ const modelGateway = {
+ url: "https://gateway.example.invalid/v1",
+ apiKey: "first-gateway-test-key",
+ apiKeyHeader: "x-gateway-key",
+ models: { [spec.id]: "target" },
+ };
+ const direct = createModelVerifier({ credentials, keyMaterial: "test-key" });
+ const gateway = createModelVerifier({ credentials, keyMaterial: "test-key", modelGateway });
+ const directOriginal = (await direct(candidate)).fingerprint;
+ const gatewayOriginal = (await gateway(candidate)).fingerprint;
+ fallback.openai = "changed-environment-test-key";
+ assert.notEqual((await direct(candidate)).fingerprint, directOriginal);
+ assert.equal((await gateway(candidate)).fingerprint, gatewayOriginal);
+ modelGateway.apiKey = "changed-gateway-test-key";
+ const rotatedGateway = (await gateway(candidate)).fingerprint;
+ assert.notEqual(rotatedGateway, gatewayOriginal);
+ modelGateway.apiKeyHeader = "x-new-gateway-key";
+ assert.notEqual((await gateway(candidate)).fingerprint, rotatedGateway);
+});
+
+import { createModelCredentialStore } from "../src/model/model-credential-store.ts";
+
+test("legacy fingerprint versions stay unavailable until a successful new verification", async () => {
+ const built = buildApp(testConfig({ openaiApiKey: "test-key" }));
+ const candidate = parseModelOverlay(spec);
+ const keyMaterial = "stable-test-key";
+ const fingerprint = createHmac("sha256", keyMaterial)
+ .update(
+ JSON.stringify({
+ version: 1,
+ spec: candidate,
+ model: modelFromOverlay(candidate),
+ key: "test-key",
+ credentialRevision: (await built.modelCredentials.statuses()).find((status) => status.provider === "openai"),
+ }),
+ )
+ .digest("hex");
+ const backing = createMemoryMap<StoredModelOverlay>();
+ await backing.put(spec.id, {
+ spec: candidate,
+ disabled: false,
+ updatedAt: 1,
+ updatedBy: "admin",
+ verification: { fingerprint, verifiedAt: 1, revision: "old-proof" },
+ });
+ let probes = 0;
+ const verifier = createModelVerifier({
+ credentials: built.modelCredentials,
+ keyMaterial,
+ probe: async () => {
+ probes++;
+ },
+ });
+ const store = createModelOverlayStore(backing, undefined, verifier);
+ await store.refresh();
+ assert.equal(probes, 0);
+ assert.equal(resolveModel(spec.id), undefined);
+ assert.match(modelUnavailableReason(spec.id)!, /verify.*again/i);
+ await store.upsert(candidate, "admin");
+ await store.refresh();
+ assert.equal(probes, 1);
+ assert.ok(resolveModel(spec.id));
+});