Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
8.1 KB
Raw Blame
import assert from "node:assert/strict";
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import { renderArtifactIndexHtml, renderPrMarkdown } from "../src/render.ts";
import { scanTestRuns } from "../src/scan.ts";
import type { TestRunRecord } from "../src/schema.ts";

function record(name: string, dir: string, createdAt: string, passed: boolean): TestRunRecord {
  return {
    name,
    dir,
    createdAt,
    closedAt: createdAt,
    engine: "v1",
    summary: {
      ok: passed,
      totalArtifacts: 1,
      passedArtifacts: passed ? 1 : 0,
      failedArtifacts: passed ? 0 : 1,
      unvalidatedArtifacts: 0,
      pendingArtifacts: 0,
      passedExpectations: passed ? 1 : 0,
      failedExpectations: passed ? 0 : 1,
      pendingJudgments: 0,
    },
    artifacts: [{
      caption: `${name} first validation`,
      fileName: "01-first.png",
      hash: `${name}-hash`,
      route: "#/workspace/test",
      at: createdAt,
      description: `${name} description`,
      model: "test-model",
      ok: passed,
      results: [{ expectation: `${name} expectation`, passed, evidence: `${name} evidence` }],
      judgments: [{ expectation: `${name} expectation`, state: passed ? "passed" : "failed", reasoning: `${name} evidence` }],
    }],
    trace: [],
    steps: [],
    outcome: passed ? "passed" : "failed",
  };
}

function legacyRecord(testRun: TestRunRecord): Record<string, unknown> {
  const frames = testRun.artifacts.map((artifact) => {
    const { judgments, ...legacyArtifact } = artifact;
    void judgments;
    return legacyArtifact;
  });
  const { engine, ...legacyTestRun } = testRun;
  void engine;
  return {
    ...legacyTestRun,
    summary: {
      ok: testRun.summary.ok,
      totalFrames: testRun.summary.totalArtifacts,
      passedFrames: testRun.summary.passedArtifacts,
      failedFrames: testRun.summary.failedArtifacts,
      unvalidatedFrames: testRun.summary.unvalidatedArtifacts,
      passedExpectations: testRun.summary.passedExpectations,
      failedExpectations: testRun.summary.failedExpectations,
    },
    frames,
    artifacts: undefined,
  };
}

test("scanTestRuns reads current and persisted legacy results while tolerating corrupt data", async () => {
  const resultsDir = await mkdtemp(join(tmpdir(), "omnirush-test-artifacts-scan-"));
  try {
    const currentDir = join(resultsDir, "test-runs", "2026-07-02T10-00-00-000Z-current");
    const oldDir = join(resultsDir, "rolls", "2026-07-01T10-00-00-000Z-old");
    const corruptDir = join(resultsDir, "test-runs", "2026-07-03T10-00-00-000Z-corrupt");
    const looseDir = join(resultsDir, "2026-06-01T10-00-00-000Z-legacy");
    await Promise.all([currentDir, oldDir, corruptDir, looseDir].map((directory) => mkdir(directory, { recursive: true })));
    await Promise.all([
      writeFile(join(currentDir, "test-run.json"), JSON.stringify(record("Current", currentDir, "2026-07-02T10:00:00.000Z", false))),
      writeFile(join(oldDir, "roll.json"), JSON.stringify(legacyRecord(record("Persisted", oldDir, "2026-07-01T10:00:00.000Z", true)))),
      writeFile(join(corruptDir, "test-run.json"), "{not-json"),
      writeFile(join(looseDir, "fraimz.html"), "legacy"),
      writeFile(join(looseDir, "frame.png"), Buffer.from("legacy-png")),
    ]);

    const entries = await scanTestRuns(resultsDir);
    assert.deepEqual(entries.map((entry) => entry.name), ["Current", "Persisted", "2026-06-01T10-00-00-000Z-legacy"]);
    const storedEntries = entries.filter((entry) => entry.kind === "test-run");
    assert.deepEqual(storedEntries.map((entry) => entry.format), ["current", "legacy"]);
    assert.equal(storedEntries[1]?.testRun.engine, "v1");
    assert.deepEqual(storedEntries[1]?.testRun.artifacts[0]?.judgments, [{
      expectation: "Persisted expectation",
      state: "passed",
      reasoning: "Persisted evidence",
    }]);

    const rendered = renderArtifactIndexHtml(entries);
    for (const expected of ["Current first validation", "FAILED", "PASSED", "current/index.html", "../rolls/", "fraimz.html"]) {
      assert.match(rendered, new RegExp(expected));
    }
  } finally {
    await rm(resultsDir, { recursive: true, force: true });
  }
});

test("renderPrMarkdown writes only the new test-evidence marker", () => {
  const testRun = record("PR proof", "/tmp/2026-pr-proof", "2026-07-02T10:00:00.000Z", false);
  testRun.engine = "v2";
  testRun.trace = [
    { seq: 1, at: testRun.createdAt, stage: "world", channel: "seed", verb: "den", detail: "den(local)", ok: true },
    { seq: 2, at: testRun.createdAt, stage: "body", channel: "user", verb: "reload", detail: "reload", ok: true },
    { seq: 3, at: testRun.createdAt, stage: "body", channel: "user", verb: "reload", detail: "reload", ok: true },
  ];
  testRun.steps = [{ seq: 1, name: "draft persists", depth: 0, ok: false, ms: 6_100, error: "missing draft" }];
  const markdown = renderPrMarkdown(testRun, { "01-first.png": "https://example.test/01-first.png" });
  assert.match(markdown, /<!-- test-evidence -->/);
  assert.doesNotMatch(markdown, /<!-- photo-roll -->|<!-- fraimz -->/);
  assert.match(markdown, /PR proof first validation/);
  assert.match(markdown, /SHA unknown · engine v2/);
  assert.match(markdown, /\*\*\[world\]\*\* den\(local\)/);
  assert.match(markdown, /\*\*\[user\]\*\* reload ×2/);
  assert.match(markdown, /\*\*steps\*\* 1 ❌ draft persists \(6\.1s\)/);
  assert.match(markdown, /\*\*verdict\*\* failed/);
});

test("renderPrMarkdown leads with the test outcome and counts trace observations", () => {
  const testRun = record("trace proof", "/tmp/2026-trace-proof", "2026-07-02T10:00:00.000Z", true);
  testRun.artifacts = [];
  testRun.summary = {
    ok: true,
    totalArtifacts: 0,
    passedArtifacts: 0,
    failedArtifacts: 0,
    unvalidatedArtifacts: 0,
    pendingArtifacts: 0,
    passedExpectations: 0,
    failedExpectations: 0,
    pendingJudgments: 0,
  };
  testRun.trace = [
    { seq: 1, at: testRun.createdAt, stage: "world", channel: "seed", verb: "tmpPath", detail: "tmpPath(ignored)", ok: true },
    { seq: 2, at: testRun.createdAt, stage: "body", channel: "user", verb: "see", detail: "see(composer, editable)", ok: true },
    { seq: 3, at: testRun.createdAt, stage: "body", channel: "user", verb: "see", detail: "see(composer, editable)", ok: true },
    { seq: 4, at: testRun.createdAt, stage: "body", channel: "user", verb: "see", detail: "see(composer, editable)", ok: true },
    { seq: 5, at: testRun.createdAt, stage: "body", channel: "probe", verb: "storage", detail: "storage(draft)", ok: true },
    { seq: 6, at: testRun.createdAt, stage: "body", channel: "probe:raw", verb: "eval", detail: "eval(<expression>)", ok: true },
  ];
  testRun.steps = [{ seq: 1, name: "draft persists", depth: 0, ok: true }];

  const markdown = renderPrMarkdown(testRun, {});
  assert.match(markdown, /^<!-- test-evidence -->\n## Test evidence — trace proof — ✅ passed/m);
  assert.match(markdown, /\*\*verdict\*\* passed · 3 user observations \(see ×3\) · 2 probes · steps 1\/1/);
  assert.doesNotMatch(markdown, /tmpPath/);

  const headings: Array<{ outcome: "failed" | "skipped"; heading: string }> = [
    { outcome: "failed", heading: "❌ failed" },
    { outcome: "skipped", heading: "⏭ skipped" },
  ];
  for (const entry of headings) {
    testRun.outcome = entry.outcome;
    assert.match(renderPrMarkdown(testRun, {}), new RegExp(`## Test evidence — trace proof — ${entry.heading}`));
  }
});

test("scanTestRuns skips a symlinked test-run.json", async () => {
  const resultsDir = await mkdtemp(join(tmpdir(), "omnirush-test-artifacts-symlink-"));
  try {
    const testRunDir = join(resultsDir, "test-runs", "2026-07-02T10-00-00-000Z-symlinked");
    await mkdir(testRunDir, { recursive: true });
    const outside = join(resultsDir, "outside-test-run.json");
    await writeFile(outside, JSON.stringify(record("Outside", testRunDir, "2026-07-02T10:00:00.000Z", true)));
    await symlink(outside, join(testRunDir, "test-run.json"));
    assert.deepEqual(await scanTestRuns(resultsDir), []);
  } finally {
    await rm(resultsDir, { recursive: true, force: true });
  }
});