Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

11.9 KB
Raw Blame
#!/usr/bin/env node
/**
 * Live native-history / opt-in takeover e2e for the OpenViking pi extension.
 *
 * This is intentionally manual: it drives a real pi binary, a real
 * OpenViking server, and a real OpenAI-compatible LLM relay.
 *
 * Required env:
 *   OPENVIKING_URL
 *   OPENVIKING_API_KEY
 *   E2E_LLM_API_KEY        (or SUPER_RELAY_API_KEY) LLM provider API key
 *
 * Optional env:
 *   PI_BIN                 path to pi binary; defaults to `which pi`
 *   E2E_LLM_BASE_URL       default https://super-relay.byted.org/v1
 *   E2E_LLM_MODEL          default model_api/experimental_0630
 *   E2E_LLM_API            pi provider api type; default openai-completions
 *   E2E_TAKEOVER=1        exercise explicit OV takeover instead of native history
 *   E2E_KEEP_TMP=1         keep temp workspace on success
 */
import { spawnSync } from "node:child_process";
import {
  copyFileSync,
  cpSync,
  existsSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  readdirSync,
  rmSync,
  writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { basename, dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

// Explicitly opt into the legacy takeover scenario; the default validates native history.
const TAKEOVER = process.env.E2E_TAKEOVER === "1";
const EXT_SRC = dirname(dirname(fileURLToPath(import.meta.url)));
const OV_URL = process.env.OPENVIKING_URL;
const OV_KEY = process.env.OPENVIKING_API_KEY;
const RELAY_KEY = process.env.E2E_LLM_API_KEY || process.env.SUPER_RELAY_API_KEY;
const LLM_BASE = process.env.E2E_LLM_BASE_URL ?? "https://super-relay.byted.org/v1";
const LLM_MODEL = process.env.E2E_LLM_MODEL ?? "model_api/experimental_0630";
const LLM_API = process.env.E2E_LLM_API ?? "openai-completions";

function which(cmd) {
  const res = spawnSync("which", [cmd], { encoding: "utf8" });
  return res.status === 0 ? res.stdout.trim() : "";
}

const PI_BIN = process.env.PI_BIN || which("pi");

for (const [key, value] of [
  ["OPENVIKING_URL", OV_URL],
  ["OPENVIKING_API_KEY", OV_KEY],
  ["E2E_LLM_API_KEY or SUPER_RELAY_API_KEY", RELAY_KEY],
  ["PI_BIN or pi on PATH", PI_BIN],
]) {
  if (!value) {
    console.error(`e2e: missing required ${key}`);
    process.exit(2);
  }
}
if (!existsSync(PI_BIN)) {
  console.error(`e2e: pi binary not found: ${PI_BIN}`);
  process.exit(2);
}

let failures = 0;
const pass = (msg) => console.log(`  PASS  ${msg}`);
const fail = (msg) => { failures++; console.error(`  FAIL  ${msg}`); };
const warn = (msg) => console.log(`  WARN  ${msg}`);
const check = (cond, msg) => (cond ? pass(msg) : fail(msg));

const root = mkdtempSync(join(tmpdir(), "ov-pi-e2e-"));
const agentDir = join(root, "agent");
const extDir = join(root, "ext");
const outDir = join(root, "out");
const projDir = join(root, "proj");
const sessionDir = join(root, "sessions");
for (const d of [agentDir, extDir, outDir, projDir, sessionDir]) {
  mkdirSync(d, { recursive: true });
}
console.log(`e2e: workspace ${root}`);

writeFileSync(join(agentDir, "models.json"), JSON.stringify({
  providers: {
    "super-relay": {
      name: "Super Relay",
      baseUrl: LLM_BASE,
      api: LLM_API,
      apiKey: "$SUPER_RELAY_API_KEY",
      ...(LLM_API === "openai-completions"
        ? { authHeader: true, headers: { "x-session-id": "ov-pi-e2e" } }
        : {}),
      models: [{
        id: LLM_MODEL,
        name: "Relay e2e model",
        reasoning: false,
        input: ["text"],
        cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
        contextWindow: 128000,
        maxTokens: 8192,
      }],
    },
  },
}, null, 2));
writeFileSync(join(agentDir, "settings.json"), JSON.stringify({ defaultProjectTrust: "always" }, null, 2));

function copyExtension() {
  for (const name of readdirSync(EXT_SRC)) {
    const src = join(EXT_SRC, name);
    const dst = join(extDir, basename(name));
    if (name.endsWith(".ts") || ["README.md", "DESIGN.md", "package.json", "package-lock.json"].includes(name)) {
      copyFileSync(src, dst);
    } else if (["lib", "shared", "scripts"].includes(name)) {
      cpSync(src, dst, { recursive: true });
    }
  }
}
copyExtension();
const install = spawnSync("npm", ["ci", "--omit=dev", "--ignore-scripts"], {
  cwd: extDir,
  timeout: 120_000,
  encoding: "utf8",
});
if (install.status !== 0) {
  console.error(`e2e: dependency install failed\n${install.stderr || install.stdout}`);
  process.exit(2);
}

// The extension has no config file of its own; its knobs live in ovcli.conf's
// plugin section like every other harness's.
const OVCLI_CONF = join(agentDir, "ovcli.conf");
writeFileSync(OVCLI_CONF, JSON.stringify({
  url: OV_URL,
  api_key: OV_KEY,
  plugin: {
    pi: {
      enabled: true,
      autoCapture: true,
      logLevel: "info",
      takeoverEnabled: TAKEOVER,
      resumeArchiveInject: false,
      commitTokenThreshold: 1000,
      commitKeepRecentCount: 0,
      takeoverTokenThreshold: 600,
      takeoverKeepRecentTurns: 1,
      takeoverOverviewBudget: 3000,
      takeoverOverviewPollMs: 3000,
      takeoverOverviewPollMax: 30,
    },
  },
}, null, 2));

const PAD1 = `PADDING-T1 ${"lorem ipsum dolor sit amet consectetur ".repeat(60)}`;
const PAD2 = `PADDING-T2 ${"vestibulum ante ipsum primis in faucibus ".repeat(60)}`;

function runTurn(turn, prompt, extraArgs = []) {
  console.log(`\ne2e: --- turn ${turn} ---`);
  const args = [
    "--provider", "super-relay", "--model", LLM_MODEL,
    "--mode", "text", "-p",
    "-ne", "-e", join(extDir, "index.ts"), "-e", join(extDir, "scripts", "e2e-probe.ts"),
    "--approve", "--no-context-files", "--no-skills", "--no-prompt-templates",
    "--session-dir", sessionDir, "--offline",
    ...extraArgs,
    prompt,
  ];
  const res = spawnSync(PI_BIN, args, {
    cwd: projDir,
    env: {
      ...process.env,
      PI_CODING_AGENT_DIR: agentDir,
      OPENVIKING_URL: OV_URL,
      OPENVIKING_API_KEY: OV_KEY,
      OPENVIKING_CLI_CONFIG_FILE: OVCLI_CONF,
      SUPER_RELAY_API_KEY: RELAY_KEY,
      OV_E2E_OUT: outDir,
      OV_E2E_TURN: String(turn),
      OV_DEBUG_LOG: join(outDir, "takeover.log"),
    },
    timeout: 300_000,
    encoding: "utf8",
  });
  const out = `${res.stdout ?? ""}`;
  const err = `${res.stderr ?? ""}`;
  if (res.status !== 0) {
    fail(`turn ${turn}: pi exited ${res.status}\nstdout:\n${out.slice(-2000)}\nstderr:\n${err.slice(-2000)}`);
  } else {
    console.log(out.trim().slice(-800));
  }
  return { out, err, status: res.status };
}

async function ovFetch(path, init) {
  const resp = await fetch(`${OV_URL}${path}`, {
    ...init,
    headers: {
      Authorization: `Bearer ${OV_KEY}`,
      "Content-Type": "application/json",
      ...(init?.headers ?? {}),
    },
  });
  const body = await resp.json().catch(() => ({}));
  return { ok: resp.ok && body?.status !== "error", body };
}

function payloadsFor(turn) {
  return readdirSync(outDir)
    .filter((f) => f.startsWith(`payload-t${turn}-`))
    .sort()
    .map((f) => readFileSync(join(outDir, f), "utf8"));
}

function toolUses(payloads) {
  const uses = [];
  for (const raw of payloads) {
    try {
      const payload = JSON.parse(raw);
      for (const message of payload.messages ?? []) {
        for (const part of Array.isArray(message.content) ? message.content : []) {
          if (part?.type === "tool_use" && typeof part.name === "string") uses.push(part);
        }
      }
    } catch {
      // A malformed probe payload is caught by the missing-tool assertions.
    }
  }
  return uses;
}

const t1 = runTurn(1, `Please remember: the release codename is ZEPHYR-9942. Just acknowledge briefly. ${PAD1}`);
const t2 = runTurn(2, `Second note: the deploy window is Friday 03:00 UTC. Acknowledge briefly. ${PAD2}`, ["-c"]);
const t3 = runTurn(3, TAKEOVER ? [
  "Recover the release codename from the archived capture.",
  "You must first call the built-in bash tool with pwd.",
  "Then follow the recovery hint in [OpenViking Session Context]: call openviking_list on the history URI,",
  "then call openviking_read on the archive messages.jsonl with explicit offset and limit.",
  "After those tool calls, answer with just the codename.",
].join(" ") : "Use bash to run pwd, then recover the release codename from our earlier conversation. Answer with the codename.", ["-c"]);

console.log("\ne2e: --- assertions ---");
check(t1.status === 0 && t2.status === 0 && t3.status === 0, "all three pi runs exited 0");

const p3 = payloadsFor(3);
check(p3.length > 0, `probe captured T3 provider payload (${p3.length} request(s))`);
if (p3.length > 0) {
  const last = p3[p3.length - 1];
  const uses = toolUses(p3);
  check(last.includes("[OpenViking Session Context]") === TAKEOVER, "OV summary injection matches explicit takeover mode");
  if (TAKEOVER) {
    // Match the full padded turn body, not the PADDING-T1 marker: the archive
    // overview and recalled memories may legitimately quote the marker.
    check(!last.includes(PAD1), "T3 request no longer contains the raw T1 turn body");
    check(last.includes("PADDING-T2") || last.includes("codename"), "T3 request keeps recent live context");
    check(uses.some((part) => part.name === "bash"), "T3 executed a built-in tool");
    check(uses.some((part) => part.name === "openviking_list"), "T3 listed the archive history");
    const reads = uses.filter((part) => part.name === "openviking_read");
    check(reads.length > 0, "T3 read the archived capture");
    check(reads.some((part) =>
      Array.isArray(part.input?.uris) &&
      Number.isFinite(part.input?.offset) &&
      Number.isFinite(part.input?.limit)
    ), "T3 used paginated archive read arguments");
  } else {
    check(last.includes("ZEPHYR-9942"), "native history retains the earlier fact");
    check(uses.some((part) => part.name === "bash"), "T3 executed a built-in tool");
  }
}

if (t3.out.includes("ZEPHYR-9942")) pass(TAKEOVER ? "model recovered the archived fact from the OV overview" : "model retained the earlier conversation fact");
else warn("model answer did not contain ZEPHYR-9942; inspect overview quality");

const sessionIdFile = join(outDir, "session-id.txt");
let ovSessionId = null;
let archiveUri = null;
const logPath = join(outDir, "takeover.log");
let takeoverLog = "";
if (existsSync(logPath)) {
  takeoverLog = readFileSync(logPath, "utf8");
  archiveUri = /boundary advanced[^\n]* via (viking:\/\/[^"\s]+)/.exec(takeoverLog)?.[1] ?? null;
}
if (existsSync(sessionIdFile)) {
  ovSessionId = `pi-${readFileSync(sessionIdFile, "utf8").trim()}`;
  const ctx = await ovFetch(`/api/v1/sessions/${encodeURIComponent(ovSessionId)}/context?token_budget=4000`);
  check(ctx.ok, `OV session ${ovSessionId} readable`);
  const result = ctx.body?.result ?? {};
  check((result.stats?.totalArchives ?? 0) >= 1, `OV session has >=1 archive (got ${result.stats?.totalArchives})`);
  if (TAKEOVER) check(Boolean(archiveUri), "takeover log identifies the exact archive URI");
  else check(!result.latest_archive_overview, "default commits do not produce a WM overview");
  if (TAKEOVER && archiveUri) {
    const overview = await ovFetch(`/api/v1/content/read?uri=${encodeURIComponent(`${archiveUri}/.overview.md`)}`);
    check(overview.ok && String(overview.body?.result ?? "").trim().length > 0, "exact archive has a non-empty overview");
  }
} else {
  fail("probe did not record a pi session id");
}

if (takeoverLog) {
  check(takeoverLog.includes("boundary advanced") === TAKEOVER, "boundary advances only in explicit takeover mode");
  console.log(`\ne2e: takeover.log:\n${takeoverLog.trim()}`);
} else {
  warn("no takeover debug log written");
}

if (ovSessionId) {
  const del = await ovFetch(`/api/v1/sessions/${encodeURIComponent(ovSessionId)}`, { method: "DELETE" });
  console.log(`\ne2e: cleanup OV session ${ovSessionId}: ${del.ok ? "deleted" : "FAILED; delete manually"}`);
}

if (failures === 0 && !process.env.E2E_KEEP_TMP) {
  rmSync(root, { recursive: true, force: true });
} else {
  console.log(`e2e: workspace kept for inspection: ${root}`);
}

console.log(failures === 0 ? "\ne2e: ALL CHECKS PASSED" : `\ne2e: ${failures} CHECK(S) FAILED`);
process.exit(failures === 0 ? 0 : 1);