Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

10.1 KB
Raw Blame
import { mountTemplate } from "./shared.ts";
import { html } from "lit";
import { classMap } from "lit/directives/class-map.js";
let config: any;
const NEW_PROVIDER = { id: "", protocol: "chat-completions", baseUrl: "", credential: "", editing: false };
const state = {
  scope: "",
  data: null as any,
  provider: { ...NEW_PROVIDER },
  rule: null as any,
  saving: false,
  message: "",
  providerStatus: false,
};
let redraw = () => {};
export function configureDecisions(options: any) {
  config = options;
}
export function loadDecisions(data: any, scope: string) {
  state.scope = scope;
  state.data = scope.startsWith("org:") ? data.decisions || null : null;
  state.rule = null;
  redraw();
}
const hookActions = (hook: string) => state.data.hooks.find((h: any) => h.id === hook)?.actions || [];
const option = (key: string, o: any, hook: string) => ({
  key,
  description: o.description || "",
  action: o.action || hookActions(hook)[0],
  message: o.message || "",
  modelId: o.runtime?.modelId || "",
  effortLevel: o.runtime?.effortLevel || "",
  fastMode: !!o.runtime?.fastMode,
});
function ruleForm(r: any) {
  const hook = r?.hook || state.data.hooks[0]?.id || "";
  return {
    editing: !!r,
    id: r?.id || "",
    hook,
    question: r?.question || "",
    provider: r?.model.provider || state.data.providers[0]?.id || "",
    model: r?.model.model || "",
    mode: r?.mode || "observe",
    onError: r?.onError || "a",
    options: Object.entries(r?.options || { a: {}, b: {} }).map(([k, o]) => option(k, o, hook)),
  };
}
function openRule(r: any) {
  state.rule = ruleForm(r);
  redraw();
  document.querySelector("#card-decisions .decision-rule")?.scrollIntoView({ block: "nearest" });
}
function ruleBody(d: any) {
  return {
    id: d.id,
    hook: d.hook,
    question: d.question,
    model: { provider: d.provider, model: d.model },
    mode: d.mode,
    onError: d.onError,
    options: Object.fromEntries(
      d.options.map((o: any) => [
        o.key,
        {
          description: o.description,
          action: o.action,
          ...(o.message ? { message: o.message } : {}),
          ...(o.action === "use_runtime"
            ? {
                runtime: {
                  modelId: o.modelId,
                  ...(o.effortLevel ? { effortLevel: o.effortLevel } : {}),
                  fastMode: o.fastMode,
                },
              }
            : {}),
        },
      ]),
    ),
  };
}
async function send(body: any, done: () => void) {
  state.providerStatus = !!body.provider;
  state.saving = true;
  state.message = "Saving…";
  redraw();
  try {
    const result = await config.save(state.scope, body);
    if (result.ok) done();
    state.message = result.ok ? "Saved." : result.error || result.data?.message || "Save failed.";
  } catch (error) {
    state.message = `Could not save: ${error instanceof Error ? error.message : String(error)}`;
  } finally {
    state.saving = false;
    redraw();
  }
}
const upsert = (list: any[], item: any) =>
  [...list.filter((x) => x.id !== item.id), item].sort((a, b) => (a.id < b.id ? -1 : 1));
function saveProvider() {
  const p = state.provider;
  const provider = {
    id: p.id,
    protocol: p.protocol,
    ...(p.baseUrl ? { baseUrl: p.baseUrl, credential: p.credential } : {}),
  };
  void send({ provider }, () => {
    state.data.providers = upsert(state.data.providers, provider);
    state.provider = { ...NEW_PROVIDER };
  });
}
function saveRule() {
  const rule = ruleBody(state.rule);
  void send({ rule }, () => {
    state.data.rules = upsert(state.data.rules, rule);
    state.rule = null;
  });
}
function remove(kind: "Provider" | "Rule", id: string) {
  if (!confirm(`Remove decision ${kind.toLowerCase()} ${id}?`)) return;
  const key = kind === "Provider" ? "providers" : "rules";
  void send({ [`remove${kind}`]: id }, () => (state.data[key] = state.data[key].filter((x: any) => x.id !== id)));
}
const field = (
  label: string,
  value: string,
  set: (v: string) => void,
  hint = "",
  locked = false,
  suggestions: readonly string[] = [],
) =>
  html`<label
    >${label}<input
      list=${suggestions.length ? `decision-${label.replace(/\s/g, "-")}` : ""}
      ?disabled=${locked}
      .value=${value}
      placeholder=${hint}
      @input=${(e: Event) => set((e.target as HTMLInputElement).value)}
    />${
      suggestions.length
        ? html`<datalist id=${`decision-${label.replace(/\s/g, "-")}`}>
            ${[...new Set(suggestions)].map((v) => html`<option value=${v}></option>`)}
          </datalist>`
        : ""
    }</label
  >`;
const select = (label: string, value: string, values: readonly string[], set: (v: string) => void) =>
  html`<label
    >${label}<select @change=${(e: Event) => (set((e.target as HTMLSelectElement).value), redraw())}>
      ${values.map((v) => html`<option value=${v} .selected=${v === value}>${v}</option>`)}
    </select></label
  >`;
const row = (label: string, onEdit: () => void, onRemove: () => void) =>
  html`<div class="feature-flag-row">
    <code>${label}</code
    ><span
      ><button ?disabled=${state.saving} @click=${onEdit}>Edit</button
      ><button class="danger" ?disabled=${state.saving} @click=${onRemove}>Remove</button></span
    >
  </div>`;
function optionFields(d: any, o: any, i: number) {
  return html`<fieldset class="decision-option">
    ${field("Option", o.key, (v) => ((o.key = v), redraw()))}
    ${field("Description", o.description, (v) => (o.description = v))}
    ${field("Message", o.message, (v) => (o.message = v), "optional")}
    ${select("Action", o.action, hookActions(d.hook), (v) => (o.action = v))}
    ${
      o.action === "use_runtime"
        ? html`${select("Model", o.modelId, ["", ...state.data.models], (v) => (o.modelId = v))}
            ${field("Effort", o.effortLevel, (v) => (o.effortLevel = v), "optional")}
            <label class="setting-toggle"
              ><input
                type="checkbox"
                .checked=${o.fastMode}
                @change=${(e: Event) => (o.fastMode = (e.target as HTMLInputElement).checked)}
              />Fast mode</label
            >`
        : ""
    }
    <button class="danger" @click=${() => (d.options.splice(i, 1), redraw())}>Remove option</button>
  </fieldset>`;
}
function ruleEditor() {
  const d = state.rule;
  const set = (key: string) => (v: string) => (d[key] = v);
  return html`<div class="decision-form decision-rule">
    ${field("Rule id", d.id, set("id"), "e.g. hard-tasks", d.editing)}
    ${select(
      "Hook",
      d.hook,
      state.data.hooks.map((h: any) => h.id),
      set("hook"),
    )}
    <label class="decision-question"
      >Question<textarea
        .value=${d.question}
        @input=${(e: Event) => (d.question = (e.target as HTMLTextAreaElement).value)}
      ></textarea>
    </label>
    ${select(
      "Decision provider",
      d.provider,
      state.data.providers.map((p: any) => p.id),
      set("provider"),
    )}
    ${field(
      "Decision model",
      d.model,
      set("model"),
      "model served by the provider",
      false,
      state.data.rules.filter((r: any) => r.model.provider === d.provider).map((r: any) => r.model.model),
    )}
    ${select("Mode", d.mode, state.data.modes, set("mode"))}
    ${select("On error", d.onError, ["", ...d.options.map((o: any) => o.key)], set("onError"))}
    ${d.options.map((o: any, i: number) => optionFields(d, o, i))}
    <div>
      <button @click=${() => (d.options.push(option("", {}, d.hook)), redraw())}>+ Option</button>
      <button class="primary" ?disabled=${state.saving} @click=${saveRule}>Save rule</button>
      <button @click=${() => ((state.rule = null), redraw())}>Cancel</button>
    </div>
  </div>`;
}
function template() {
  const data = state.data;
  const p = state.provider;
  return html`<section class=${classMap({ card: true, "sv-customize": true, hidden: !data })} id="card-decisions">
    <div class="head">
      <h2>Decision rules</h2>
      <p>
        Classifiers that run at harness hooks. A rule asks its decision model a question, picks an option, and applies
        that option's action. Observe rules only log their verdicts.
      </p>
    </div>
    ${
      data
        ? html`<div class="body">
              <h3>Providers</h3>
              ${data.providers.map((x: any) =>
                row(
                  `${x.id}: ${x.protocol}${x.baseUrl ? ` · ${x.baseUrl} · ${x.credential}` : " · model gateway"}`,
                  () => ((state.provider = { ...NEW_PROVIDER, ...x, editing: true }), redraw()),
                  () => remove("Provider", x.id),
                ),
              )}
              <div class="decision-form">
                ${field("Provider id", p.id, (v) => (p.id = v), "", p.editing)}
                ${select("Protocol", p.protocol, data.protocols, (v) => (p.protocol = v))}
                ${field("Base URL", p.baseUrl, (v) => (p.baseUrl = v), "blank uses the model gateway")}
                ${field("Credential", p.credential, (v) => (p.credential = v), "org service credential", false, data.credentials || [])}
                <button ?disabled=${state.saving || !p.id} @click=${saveProvider}>Save provider</button>
                ${p.editing ? html`<button @click=${() => ((state.provider = { ...NEW_PROVIDER }), redraw())}>Cancel</button>` : ""}
                <span class="status">${state.providerStatus ? state.message : ""}</span>
              </div>
              <h3>Rules</h3>
              ${data.rules.map((r: any) =>
                row(
                  `${r.id}: ${r.hook} · ${r.mode}`,
                  () => openRule(r),
                  () => remove("Rule", r.id),
                ),
              )}
              ${state.rule ? ruleEditor() : ""}
            </div>
            <div class="foot">
              <button
                class="primary"
                ?disabled=${state.saving || !!state.rule || !data.providers.length}
                @click=${() => openRule(null)}
              >
                + Add rule</button
              ><span class="status">${state.providerStatus ? "" : state.message}</span>
            </div>`
        : ""
    }
  </section>`;
}
export function mountDecisions() {
  redraw = mountTemplate('template[data-settings-card="card-decisions"]', template);
}