Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

7.6 KB
Raw Blame
import { sendJson } from "../../http.ts";
import { audit, authorizeAdmin, orgScope } from "../shared.ts";
import type { ApiCtx } from "../route.ts";
import { securityScreenSystemPrompt, type SecurityScreenVerdict } from "../../../security/security-posture.ts";
import { defaultAutoFlaggerConfig, parseAutoFlaggerDraft, type AutoFlaggerDraft } from "../admin-resources.ts";
import { isHarnessId } from "../../../model/pi-models.ts";
import type { ScreenSample } from "../../../sessions/session-store.ts";
import type { SecurityScreenProbe } from "../../../security/security-screener.ts";

const DEFAULT_WINDOW = 100;
const MAX_WINDOW = 500;
const CONCURRENCY = 4;
const PER_SAMPLE_TIMEOUT_MS = 20_000;
const RUN_DEADLINE_MS = 120_000;

interface RunTotals {
  flagged: number;
  allowed: number;
  unscreened: number;
  errors: number;
}

const emptyTotals = (): RunTotals => ({ flagged: 0, allowed: 0, unscreened: 0, errors: 0 });

const rate = (flagged: number, scored: number): number | null =>
  scored > 0 ? Math.round((flagged / scored) * 10_000) / 10_000 : null;

/**
 * Replay past screenings through one flagger configuration. Verdicts are counted, never returned —
 * the payloads belong to the conversations they came from, and the answer here is a rate.
 */
async function replay(
  probe: SecurityScreenProbe,
  config: AutoFlaggerDraft,
  samples: readonly ScreenSample[],
  actorId: string,
  deadline: number,
): Promise<{ totals: RunTotals; verdicts: Map<string, "strict" | "auto">; complete: boolean }> {
  const systemPrompt = securityScreenSystemPrompt(config.rubric);
  const totals = emptyTotals();
  const verdicts = new Map<string, "strict" | "auto">();
  let next = 0;
  let complete = true;
  const worker = async (): Promise<void> => {
    for (;;) {
      if (Date.now() >= deadline) {
        complete = false;
        return;
      }
      const sample = samples[next++];
      if (!sample) return;
      let verdict: SecurityScreenVerdict | undefined;
      try {
        verdict = await probe({
          payload: sample.payload,
          harnessId: config.harnessId,
          modelId: config.modelId,
          systemPrompt,
          actorId,
          scopeLabel: sample.scopeLabel,
          signal: AbortSignal.timeout(PER_SAMPLE_TIMEOUT_MS),
        });
      } catch {
        verdict = undefined;
      }
      if (!verdict) totals.errors += 1;
      else if (verdict.unscreened) totals.unscreened += 1;
      else if (verdict.decision === "strict") {
        totals.flagged += 1;
        verdicts.set(sample.id, "strict");
      } else {
        totals.allowed += 1;
        verdicts.set(sample.id, "auto");
      }
    }
  };
  await Promise.all(Array.from({ length: Math.min(CONCURRENCY, samples.length) }, worker));
  return { totals, verdicts, complete };
}

/**
 * POST /v1/admin/scopes/:scope/auto-flagger/test — run a candidate flagger over the most recent
 * real screenings and report how often it flags. Org-wide, admin-only, aggregates only.
 */
export async function testAutoFlagger(ctx: ApiCtx): Promise<void> {
  const scope = orgScope();
  const requestedScope = decodeURIComponent(ctx.params.scope ?? scope);
  const actor = await authorizeAdmin(ctx, scope);
  if (!actor) return;
  if (requestedScope !== scope) {
    sendJson(ctx.res, 400, { error: "bad_request", message: "the Auto flagger is org-wide; request an org scope" });
    return;
  }
  await ctx.deps.refreshModels?.();
  const body = (ctx.body ?? {}) as {
    window?: unknown;
    harnessId?: unknown;
    modelId?: unknown;
    rubric?: unknown;
    compare?: unknown;
  };
  const requestedWindow = body.window === undefined ? DEFAULT_WINDOW : Number(body.window);
  if (!Number.isInteger(requestedWindow) || requestedWindow < 1 || requestedWindow > MAX_WINDOW) {
    sendJson(ctx.res, 400, {
      error: "bad_request",
      message: `window must be an integer between 1 and ${MAX_WINDOW}`,
    });
    return;
  }
  const { sessions, screenSecurity, config: configStore } = ctx.deps;
  if (!sessions || !screenSecurity || !configStore) {
    sendJson(ctx.res, 501, {
      error: "unsupported",
      message: "this deployment has no screening model wired, so the flagger cannot be tested",
    });
    return;
  }
  const saved = configStore.getAutoFlaggerConfig();
  const fallback = defaultAutoFlaggerConfig(ctx.deps);
  const effective: AutoFlaggerDraft =
    saved && isHarnessId(saved.harnessId)
      ? { harnessId: saved.harnessId, modelId: saved.modelId, rubric: saved.rubric }
      : fallback;
  const draftGiven = body.harnessId !== undefined || body.modelId !== undefined || body.rubric !== undefined;
  const parsed = draftGiven
    ? await parseAutoFlaggerDraft(ctx.deps, body)
    : ({ value: effective } as { value: AutoFlaggerDraft });
  if ("error" in parsed) {
    sendJson(ctx.res, 400, { error: "bad_request", message: parsed.error });
    return;
  }
  const candidate = parsed.value;
  const samples = await sessions.listScreenSamples(requestedWindow);
  const startedAt = Date.now();
  const deadline = startedAt + RUN_DEADLINE_MS;
  if (!samples.length) {
    audit(ctx.deps, {
      principalId: actor.id,
      action: "auto_flagger.test",
      resource: "auto-flagger",
      scopeLabel: scope,
      status: "empty",
      detail: JSON.stringify({ window: requestedWindow, sampled: 0 }),
    });
    sendJson(ctx.res, 200, {
      scopeId: scope,
      window: requestedWindow,
      sampled: 0,
      harnessId: candidate.harnessId,
      modelId: candidate.modelId,
      message: "no past screenings are recorded yet, so there is nothing to replay",
    });
    return;
  }
  const run = await replay(screenSecurity, candidate, samples, actor.id, deadline);
  const scored = run.totals.flagged + run.totals.allowed;
  const wantCompare =
    body.compare === true &&
    (candidate.harnessId !== effective.harnessId ||
      candidate.modelId !== effective.modelId ||
      candidate.rubric !== effective.rubric);
  const baseline = wantCompare ? await replay(screenSecurity, effective, samples, actor.id, deadline) : null;
  let changed: number | null = null;
  if (baseline) {
    changed = 0;
    for (const [id, verdict] of run.verdicts) {
      const before = baseline.verdicts.get(id);
      if (before && before !== verdict) changed += 1;
    }
  }
  const result = {
    scopeId: scope,
    window: requestedWindow,
    sampled: samples.length,
    scored,
    flagged: run.totals.flagged,
    allowed: run.totals.allowed,
    unscreened: run.totals.unscreened,
    errors: run.totals.errors,
    flagRate: rate(run.totals.flagged, scored),
    oldestAt: samples[samples.length - 1]?.createdAt ?? null,
    newestAt: samples[0]?.createdAt ?? null,
    harnessId: candidate.harnessId,
    modelId: candidate.modelId,
    saved: !!saved,
    partial: !run.complete || !(baseline?.complete ?? true),
    durationMs: Date.now() - startedAt,
    ...(baseline
      ? {
          baseline: {
            harnessId: effective.harnessId,
            modelId: effective.modelId,
            flagged: baseline.totals.flagged,
            flagRate: rate(baseline.totals.flagged, baseline.totals.flagged + baseline.totals.allowed),
            changed,
          },
        }
      : {}),
  };
  audit(ctx.deps, {
    principalId: actor.id,
    action: "auto_flagger.test",
    resource: "auto-flagger",
    scopeLabel: scope,
    status: result.partial ? "partial" : "ok",
    detail: JSON.stringify({
      window: requestedWindow,
      sampled: result.sampled,
      flagged: result.flagged,
      errors: result.errors,
      harnessId: candidate.harnessId,
      modelId: candidate.modelId,
      compared: !!baseline,
    }),
  });
  sendJson(ctx.res, 200, result);
}