Skip to content

The open-source app everyone uses to manage agents at work

22.3 KB
Raw Blame
import { isValidNativePrpEnvelope } from "./native-event-envelope.js";
import { nativeSha256 } from "../../server/src/services/native-runtime/canonical.js";
import { REDACTED_EVENT_VALUE } from "../../server/src/redaction.js";
import { CASE_RECEIPT_BUDGET_CENTS, CASE_RECEIPT_MAX_TURNS, CASE_RECEIPT_RUN_TIMEOUT_MS, type CaseReceiptAgents } from "./case-receipt-cases.js";

type Row = Record<string, any>;
export interface CaseReceiptObservation {
  nonce: string; companyId: string; taskId: string; caseId: string; agents: CaseReceiptAgents;
  issue: Row; caseRecord: Row; cases: Row[]; tasks: Row[]; handoffs: Row[]; runs: Row[];
  receipt: Row | null; receiptRevisions: Row[]; review: Row | null;
  initialBrief?: Row | null; triageReadBrief?: boolean;
  handoffToolRunIds: string[]; verifierReadReceipt: boolean; builderReadReview: boolean; kickoffReplayed: boolean;
  pendingInteractions: number;
  budgets?: { company: Row; agents: Row[] };
  configuredLimits?: { maxRuns: number; runTimeoutMs: number; attemptTimeoutMs: number; automaticRetryPolicy?: string; agents: Row[] };
}

const object = (value: unknown): Row => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Row : {};
const document = (value: unknown) => { const row = object(value); return object(row.document ?? row); };
const body = (value: unknown) => {
  const row = document(value);
  try { return typeof (row.body ?? row.latestBody) === "string" ? JSON.parse(row.body ?? row.latestBody) : null; } catch { return null; }
};
const timestamp = (value: unknown) => typeof value === "string" || value instanceof Date ? Date.parse(String(value)) : NaN;
const integer = (value: unknown): value is number => typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
const id = (value: unknown): value is string => typeof value === "string" && value.length > 0;
const keys = (value: unknown, expected: string[]) => {
  const actual = Object.keys(object(value)).sort();
  return actual.length === expected.length && actual.every((key, index) => key === [...expected].sort()[index]);
};
const initialReceipt = (value: unknown) => keys(value, ["total"]) && object(value).total === 6;
const finalReceipt = (value: unknown) => keys(value, ["total", "currency"]) && object(value).total === 6 && object(value).currency === "USD";
const expectedReview = (value: unknown) => keys(value, ["totalCorrect", "missing"]) && object(value).totalCorrect === true
  && Array.isArray(object(value).missing) && object(value).missing.length === 1 && object(value).missing[0] === "currency";

/** Ordinary native document tools are scoped to the admitted task. The saved
 * result must identify that task, document and revision, and correlate with a
 * successful runner execution. Model prose cannot establish a persisted read. */
export function hasTaskDocumentRead(events: readonly Row[], taskId: string, key: string, requireBeforeDocumentWrite = false): boolean {
  const name = (value: unknown) => typeof value === "string" ? value.split(".").at(-1) : undefined;
  const valid = events.flatMap(event => {
    const envelope = object(event.payload).prpEvent;
    return envelope?.sourceKind === "runner" && isValidNativePrpEnvelope(envelope, event.protocolSchemaVersion)
      && integer(event.seq) ? [{ event, payload: object(envelope.payload) }] : [];
  });
  const savedDocument = (result: unknown) => {
    const value = object(result);
    return object(object(value.result ?? value).document);
  };
  const persisted = (saved: Row) => saved.issueId === taskId && id(saved.id) && id(saved.latestRevisionId)
    && integer(saved.latestRevisionNumber) && saved.latestRevisionNumber > 0;
  const completed = (callId: string, tool: string, after: number, before = Infinity) => valid.some(({ event, payload }) =>
    event.seq > after && event.seq < before && event.eventType === "tool.execution.completed"
    && name(payload.name) === tool && payload.executionId === callId && payload.status === "completed");
  const firstWrite = requireBeforeDocumentWrite ? valid.filter(({ event, payload }) => event.eventType === "item.started"
    && payload.item?.type === "tool_use" && name(payload.item.name) === "write_document")
    .sort((a, b) => a.event.seq - b.event.seq)[0] : undefined;
  if (requireBeforeDocumentWrite) {
    if (!firstWrite) return false;
    const item = object(firstWrite.payload.item), input = object(item.input);
    if (!id(item.id) || !id(input.key) || !completed(item.id, "write_document", firstWrite.event.seq)) return false;
    if (!valid.some(({ event, payload }) => {
      const returned = object(payload.item), saved = savedDocument(returned.result);
      return event.seq > firstWrite.event.seq && event.eventType === "item.completed" && returned.type === "tool_result"
        && returned.id === item.id && (returned.tool_use_id === undefined || returned.tool_use_id === item.id)
        && persisted(saved) && saved.key === input.key;
    })) return false;
  }
  return valid.some(({ event: start, payload }) => {
    const item = object(payload.item), input = object(item.input);
    if (start.eventType !== "item.started" || item.type !== "tool_use" || name(item.name) !== "read_document"
      || !id(item.id) || !keys(input, ["key"]) || input.key !== key
      || !completed(item.id, "read_document", start.seq, firstWrite?.event.seq)) return false;
    return valid.some(({ event, payload: returned }) => {
      const resultItem = object(returned.item), saved = savedDocument(resultItem.result);
      return event.seq > start.seq && (!firstWrite || event.seq < firstWrite.event.seq) && event.eventType === "item.completed"
        && resultItem.type === "tool_result" && resultItem.id === item.id
        && (resultItem.tool_use_id === undefined || resultItem.tool_use_id === item.id)
        && persisted(saved) && saved.key === key && typeof (saved.latestBody ?? saved.body) === "string";
    });
  });
}

/** Historical reader retained for immutable earlier case-document campaigns. */
export function hasCaseDocumentRead(events: readonly Row[], caseId: string, key: string, requireBeforeDocumentWrite = false): boolean {
  const operation = "GET /api/cases/{id}/documents/{key}";
  const toolName = (name: unknown) => typeof name === "string" ? name.split(".").at(-1) : undefined;
  const apiTool = (name: unknown) => toolName(name) === "call_api";
  const readTool = (name: unknown) => apiTool(name) || toolName(name) === "read_case_document";
  const writeTool = (name: unknown) => apiTool(name) || toolName(name) === "write_case_document";
  const expectedInput = { operationId: operation, pathParams: { id: caseId, key } };
  // Production history hides API parameters and bodies. Its canonical request
  // digest can still prove this exact public read without exposing the content.
  const inputHashes = new Set([nativeSha256(expectedInput), nativeSha256({ ...expectedInput, query: {} })]);
  const nativeInputHash = nativeSha256({ caseId, key });
  const receiptFields = new Set(["operationId", "apiOperationId", "callId", "ok", "status", "contentType", "data", "apiPayloadRedacted", "apiPayloadSha256"]);
  const redacted = (value: Row) => value.apiPayloadRedacted === true && value.data === REDACTED_EVENT_VALUE
    && typeof value.apiPayloadSha256 === "string" && /^[a-f0-9]{64}$/.test(value.apiPayloadSha256)
    && Object.keys(value).every(field => receiptFields.has(field));
  const hasRedaction = (value: Row) => "apiPayloadRedacted" in value || "apiPayloadSha256" in value;
  const valid = events.flatMap(event => {
    const envelope = object(event.payload).prpEvent;
    return envelope?.sourceKind === "runner" && isValidNativePrpEnvelope(envelope, event.protocolSchemaVersion)
      && integer(event.seq) ? [{ event, payload: object(envelope.payload) }] : [];
  });
  const writeOperation = "PUT /api/cases/{id}/documents/{key}";
  const firstWrite = requireBeforeDocumentWrite ? valid.filter(({ event, payload }) => {
    const item = object(payload.item);
    return event.eventType === "item.started" && item.type === "tool_use" && writeTool(item.name)
      && (!apiTool(item.name) || object(item.input).operationId === writeOperation);
  }).sort((a, b) => a.event.seq - b.event.seq)[0] : undefined;
  if (requireBeforeDocumentWrite) {
    if (!firstWrite) return false;
    const item = object(firstWrite.payload.item), input = object(item.input);
    const writeParams = object(input.pathParams);
    const validInput = hasRedaction(input) ? redacted(input)
      : apiTool(item.name) ? id(writeParams.id) && id(writeParams.key) : id(input.caseId) && id(input.key);
    if (!id(item.id) || !validInput) return false;
    const completed = valid.some(({ event, payload }) => event.seq > firstWrite.event.seq
      && event.eventType === "tool.execution.completed" && toolName(payload.name) === toolName(item.name)
      && payload.executionId === item.id && payload.status === "completed");
    const succeeded = valid.some(({ event, payload }) => {
      const returned = object(payload.item), result = object(returned.result);
      const saved = object(result.data);
      const validResult = hasRedaction(result) ? redacted(result) : id(object(saved.document).id) && id(object(saved.revision).id);
      return event.seq > firstWrite.event.seq && event.eventType === "item.completed" && returned.type === "tool_result"
        && returned.id === item.id && (returned.tool_use_id === undefined || returned.tool_use_id === item.id)
        && result.ok === true && result.status === 200 && result.apiOperationId === writeOperation
        && validResult;
    });
    if (!completed || !succeeded) return false;
  }
  for (const { event: start, payload } of valid) {
    const item = object(payload.item), input = object(item.input), params = object(input.pathParams);
    const matchesInput = hasRedaction(input) ? redacted(input) && (apiTool(item.name)
      ? inputHashes.has(input.apiPayloadSha256) : input.apiPayloadSha256 === nativeInputHash)
      : apiTool(item.name) ? params.id === caseId && params.key === key
        : keys(input, ["caseId", "key"]) && input.caseId === caseId && input.key === key;
    if (start.eventType !== "item.started" || item.type !== "tool_use" || !readTool(item.name) || !id(item.id)
      || (apiTool(item.name) && input.operationId !== operation) || !matchesInput) continue;
    const completed = valid.some(({ event, payload: finished }) => event.seq > start.seq && event.eventType === "tool.execution.completed"
      && toolName(finished.name) === toolName(item.name) && finished.executionId === item.id && finished.status === "completed");
    if (!completed) continue;
    if (valid.some(({ event, payload: returned }) => {
      const resultItem = object(returned.item), result = object(resultItem.result), saved = object(result.data);
      const matchesResult = hasRedaction(result) ? redacted(result)
        : id(saved.id) && saved.key === key && id(saved.latestRevisionId) && typeof (saved.body ?? saved.latestBody) === "string";
      return event.seq > start.seq && event.eventType === "item.completed" && resultItem.type === "tool_result"
        && (!firstWrite || event.seq < firstWrite.event.seq)
        && resultItem.id === item.id && (resultItem.tool_use_id === undefined || resultItem.tool_use_id === item.id)
        && result.ok === true && result.status === 200 && result.apiOperationId === operation
        && matchesResult;
    })) return true;
  }
  return false;
}

export function gradeCaseReceipt(input: CaseReceiptObservation) {
  const checks: Array<{ id: string; passed: boolean; detail: string }> = [];
  const add = (name: string, passed: boolean, detail: string) => checks.push({ id: name, passed, detail });
  const issue = object(input.issue), workflow = object(input.caseRecord), fields = object(workflow.fields);
  const tasks = Array.isArray(input.tasks) ? input.tasks : [], cases = Array.isArray(input.cases) ? input.cases : [];
  const runs = [...(Array.isArray(input.runs) ? input.runs : [])].sort((a, b) => timestamp(a.startedAt) - timestamp(b.startedAt));
  const handoffs = [...(Array.isArray(input.handoffs) ? input.handoffs : [])].sort((a, b) => a.version - b.version);
  const phases = ["triage", "implementation", "verification", "revision"];
  const agents = [input.agents?.triage, input.agents?.implementation, input.agents?.verification, input.agents?.implementation];
  const distinctAgents = [...new Set(agents)];
  const passives = cases.filter(row => row.id !== input.caseId);
  const listedWorkflow = cases.filter(row => row.id === input.caseId);
  add("one-primary-task-and-two-passive-cases", id(input.nonce) && distinctAgents.length === 3 && distinctAgents.every(id)
    && issue.id === input.taskId && issue.companyId === input.companyId && tasks.length === 1 && tasks[0]?.id === input.taskId
    && workflow.id === input.caseId && workflow.companyId === input.companyId && workflow.workIssueId === input.taskId
    && cases.length === 3 && new Set(cases.map(row => row.id)).size === 3 && listedWorkflow.length === 1
    && listedWorkflow[0]?.workIssueId === input.taskId && passives.length === 2
    && passives.every(row => row.companyId === input.companyId && row.caseType === "receipt-research" && row.workIssueId === null
      && row.phase === null && row.handoffVersion === 0 && row.fields?.nonce === input.nonce)
    && passives.map(row => row.status).sort().join(",") === "done,draft",
  "Exactly one company task and workflow case exist; the two draft/done research cases remain passive.");
  add("unchanged-receipt-inputs", fields.nonce === input.nonce && fields.currency === "USD" && Array.isArray(fields.items)
    && fields.items.length === 1 && keys(fields.items[0], ["quantity", "unitPrice"])
    && fields.items[0].quantity === 2 && fields.items[0].unitPrice === 3,
  "The persisted fixture inputs independently imply total 6 and currency USD.");
  add("triage-validation-persisted", fields.inputsValidated === true,
    "Triage saved its input validation through the native case update surface.");
  const brief = object(input.initialBrief);
  add("initial-task-brief-read", brief.issueId === input.taskId && brief.key === "brief" && brief.latestRevisionNumber === 1
    && (brief.latestBody ?? brief.body) === `Receipt ${input.nonce}: total and currency are required.` && input.triageReadBrief === true,
  "The initial brief belongs to the primary task and triage read its persisted revision through the ordinary native document tool.");
  const validRuns = runs.length === CASE_RECEIPT_MAX_TURNS && new Set(runs.map(run => run.id)).size === CASE_RECEIPT_MAX_TURNS
    && runs.every((run, index) => id(run.id) && run.companyId === input.companyId && run.agentId === agents[index]
      && run.runtimeMode === "native" && run.status === "succeeded"
      && [run.issueId, run.nativeIssueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].includes(input.taskId)
      && Number.isFinite(timestamp(run.startedAt)) && Number.isFinite(timestamp(run.finishedAt))
      && timestamp(run.finishedAt) >= timestamp(run.startedAt)
      && timestamp(run.finishedAt) - timestamp(run.startedAt) <= CASE_RECEIPT_RUN_TIMEOUT_MS
      && (index === 0 || timestamp(run.startedAt) >= timestamp(runs[index - 1]?.finishedAt)));
  add("four-settled-nonoverlapping-runs", validRuns,
    "Exactly four successful native task runs follow triage, builder, verifier, returning builder without overlap or an extra turn.");
  const validHandoffs = handoffs.length === 5 && new Set(handoffs.map(row => row.id)).size === 5
    && handoffs.slice(0, 4).every((row, index) => row.destinationKind === "agent" && id(row.id) && row.companyId === input.companyId && row.caseId === input.caseId
      && row.issueId === input.taskId && row.version === index + 1 && row.phase === phases[index] && row.toAgentId === agents[index]
      && row.state === "delivered" && integer(row.appliedIssueStatusVersion)
      && (index === 0 ? row.sourceRunId === null && (row.fromAgentId === null || row.fromAgentId === agents[0])
        : row.fromAgentId === agents[index - 1] && row.sourceRunId === runs[index - 1]?.id)
      && Number.isFinite(timestamp(row.assignmentAppliedAt)) && Number.isFinite(timestamp(row.deliveredAt))
      && timestamp(row.assignmentAppliedAt) <= timestamp(runs[index]?.startedAt)
      && timestamp(row.deliveredAt) >= timestamp(row.assignmentAppliedAt)
      && (index === 0 || timestamp(row.assignmentAppliedAt) >= timestamp(runs[index - 1]?.finishedAt)));
  add("same-task-ordered-handoffs", validRuns && validHandoffs,
    "Four delivered, run-attributed routing revisions keep the original task and wait for each outgoing run to finish.");
  const toolRuns = Array.isArray(input.handoffToolRunIds) ? input.handoffToolRunIds : [];
  add("native-handoff-receipts-and-kickoff-replay", input.kickoffReplayed === true && toolRuns.length === 4
    && new Set(toolRuns).size === 4 && runs.every(run => toolRuns.includes(run.id)) && validRuns && validHandoffs,
  "Kickoff replay preserved the receipt, and all agent transfers and the terminal handoff have completed native tool receipts.");
  add("fresh-phase-context", validRuns && validHandoffs && runs.every((run, index) => {
    const context = object(run.contextSnapshot?.paperclipCase);
    return context.id === input.caseId && context.taskId === input.taskId && context.phase === phases[index]
      && context.handoffVersion === index + 1 && integer(context.issueStatusVersion)
      && context.issueStatusVersion >= handoffs[index]!.appliedIssueStatusVersion
      && context.incomingHandoff?.id === handoffs[index]!.id
      && typeof context.incomingHandoff?.instructions === "string" && context.incomingHandoff.instructions.includes(input.nonce);
  }), "Every admitted run received its own current handoff ID, phase, routing version, and observed task version.");

  const receipt = document(input.receipt), review = document(input.review), reviewRevision = object(input.review?.revision);
  const revisions = [...(Array.isArray(input.receiptRevisions) ? input.receiptRevisions : [])].sort((a, b) => a.revisionNumber - b.revisionNumber);
  const inRun = (revision: Row, run: Row | undefined) => !!run && revision.createdByRunId === run.id
    && timestamp(revision.createdAt) >= timestamp(run.startedAt) && timestamp(revision.createdAt) <= timestamp(run.finishedAt);
  add("persisted-receipt-correction", validRuns && id(receipt.id) && receipt.companyId === input.companyId
    && receipt.createdByAgentId === input.agents.implementation && receipt.updatedByAgentId === input.agents.implementation
    && revisions.length === 2 && revisions.every((revision, index) => id(revision.id) && revision.revisionNumber === index + 1
      && revision.createdByAgentId === input.agents.implementation && revision.createdByUserId == null)
    && revisions[0]?.id !== revisions[1]?.id && inRun(revisions[0] ?? {}, runs[1]) && inRun(revisions[1] ?? {}, runs[3])
    && initialReceipt(body(revisions[0])) && finalReceipt(body(revisions[1])) && finalReceipt(body(receipt))
    && receipt.latestRevisionId === revisions[1]?.id && receipt.latestRevisionNumber === 2,
  "The first builder saved only total 6; the same returning builder revised that task document to total 6 and currency USD.");
  add("independent-persisted-review", validRuns && id(review.id) && review.id !== receipt.id && review.companyId === input.companyId
    && review.createdByAgentId === input.agents.verification && review.updatedByAgentId === input.agents.verification
    && review.latestRevisionId === reviewRevision.id && review.latestRevisionNumber === 1 && reviewRevision.revisionNumber === 1
    && id(reviewRevision.id) && reviewRevision.createdByAgentId === input.agents.verification && reviewRevision.createdByUserId == null
    && inRun(reviewRevision, runs[2]) && expectedReview(body(review)) && expectedReview(body(reviewRevision))
    && timestamp(revisions[0]?.createdAt) < timestamp(reviewRevision.createdAt)
    && timestamp(reviewRevision.createdAt) < timestamp(revisions[1]?.createdAt),
  "A separate verifier persisted the actual total comparison and missing currency before the returning builder corrected the receipt.");
  add("persisted-document-read-receipts", input.verifierReadReceipt === true && input.builderReadReview === true,
    "Correlated successful native API receipts prove the verifier read receipt and the returning builder read review.");
  const terminal = handoffs[4];
  add("explicit-terminal-case-completion", validRuns && validHandoffs && issue.status === "done"
    && issue.assigneeAgentId === input.agents.implementation && workflow.status === "done"
    && workflow.phase === "done" && workflow.handoffVersion === 5 && workflow.pendingHandoff == null
    && input.pendingInteractions === 0 && Number.isFinite(timestamp(workflow.completedAt))
    && terminal?.destinationKind === "done" && terminal.state === "completed" && terminal.phase === "done"
    && terminal.companyId === input.companyId && terminal.caseId === input.caseId && terminal.issueId === input.taskId
    && terminal.version === 5 && terminal.fromAgentId === input.agents.implementation && terminal.toAgentId === null
    && terminal.sourceRunId === runs[3]?.id && Number.isFinite(timestamp(terminal.completedAt))
    && timestamp(terminal.completedAt) === timestamp(workflow.completedAt)
    && timestamp(terminal.completedAt) >= timestamp(runs[3]?.startedAt)
    && timestamp(terminal.completedAt) <= timestamp(runs[3]?.finishedAt),
  "The returning builder explicitly handed off to done; accepted completion closed the same task and case without another run.");

  const limits = input.configuredLimits, budgets = input.budgets, roleLimits = [1, 2, 1];
  const persistedAgents = Array.isArray(limits?.agents) ? limits.agents : [], budgetAgents = Array.isArray(budgets?.agents) ? budgets.agents : [];
  add("configured-four-turn-budget", limits?.maxRuns === 4 && limits.runTimeoutMs === CASE_RECEIPT_RUN_TIMEOUT_MS
    && limits.attemptTimeoutMs === 5 * 60_000 && limits.automaticRetryPolicy === "single_attempt"
    && budgets?.company?.budgetMonthlyCents === CASE_RECEIPT_BUDGET_CENTS && budgetAgents.length === 3 && persistedAgents.length === 3
    && distinctAgents.every((agentId, index) => budgetAgents.filter(agent => agent.id === agentId && agent.budgetMonthlyCents === CASE_RECEIPT_BUDGET_CENTS).length === 1
      && persistedAgents.filter(agent => agent.id === agentId && agent.maxDailyRuns === roleLimits[index] && agent.maxConcurrentRuns === 1
        && agent.heartbeatEnabled === false && agent.timeoutSec === 90 && agent.modelReasoningEffort === "low").length === 1),
  "Public readback confirms 100-cent company/agent stops, role run caps 1/2/1, one concurrent run, low effort, 90-second turns, and one five-minute attempt.");
  return checks;
}