Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
#!/usr/bin/env node
/**
* Live e2e gate for the agent-driven context window.
*
* Manual by design: it drives a real pi binary, a real OpenViking server and a
* real OpenAI-compatible LLM relay, and asserts the whole reset pipeline —
* archive to OpenViking, virtual cut of the provider payload, frozen window
* header, cross-process restore, fail-closed refusal.
*
* Scenario (plan "端到端"):
* T1 `pi -p` remember ZEPHYR-9942 + the deploy window, padded with PAD1.
* T2 `pi -c` instruct the model to call new_context alone with a fixed
* reason and notes carrying the codename and the next step,
* then reply READY in the new window.
* T3 `pi -c` ask for the codename (a fresh process, so the window state
* has to survive in the session file).
*
* Required env:
* OPENVIKING_URL
* OPENVIKING_API_KEY
* E2E_LLM_API_KEY LLM provider API key (never written to disk)
*
* Optional env:
* PI_BIN path to the pi binary; defaults to `which pi`
* E2E_LLM_BASE_URL base URL of an OpenAI- or Anthropic-compatible
* endpoint; required, no default, so a private relay
* never ends up hardcoded here
* E2E_LLM_MODEL model id served by that endpoint; required
* E2E_LLM_API pi provider api type; default openai-completions
* E2E_LLM_REASONING thinking level for the model: off (default),
* minimal, low, medium, high or xhigh. Anything but
* off marks the model as reasoning-capable and sets
* pi's defaultThinkingLevel, so an OpenAI-compatible
* relay receives reasoning_effort.
* E2E_KEEP_TMP=1 keep the temp workspaces even on success
* E2E_KEEP_OV_SESSION=1 do not delete the OpenViking sessions afterwards,
* so the archives stay readable on the server (for
* demos; the sessions are yours to clean up)
* E2E_WINDOW_LONG=1 run ONLY the long-context scenario: the agent
* inventories a real code tree until context
* pressure makes it reset on its own, with no
* instruction to call the tool. `=both` adds it to
* the other scenarios. Slow (10-20 min) and it
* depends on model judgement, so the judgement
* checks warn rather than fail.
* E2E_WINDOW_FAILCLOSED=1 run ONLY the fail-closed variant: OPENVIKING_URL
* is pointed at a closed port and the gate asserts
* that nothing was cut. `=both` runs both scenarios.
*
* Exit codes: 0 all checks passed, 1 a check failed, 2 bad environment.
*/
import { spawnSync } from "node:child_process";
import {
copyFileSync,
cpSync,
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
readdirSync,
statSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { basename, dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
// The gate asserts on the extension's own markers, so it imports them instead
// of copying the strings: a rename in the core has to break the gate loudly.
import {
HANDOFF_MARKER,
STATUS_CUSTOM_TYPE,
WINDOW_ENTRY_TYPE,
WINDOW_HEADER_OPEN,
} from "../lib/context-window-core.mjs";
const EXT_SRC = dirname(dirname(fileURLToPath(import.meta.url)));
const OV_URL = process.env.OPENVIKING_URL;
const OV_KEY = process.env.OPENVIKING_API_KEY;
const LLM_KEY = process.env.E2E_LLM_API_KEY;
const LLM_BASE = process.env.E2E_LLM_BASE_URL;
const LLM_MODEL = process.env.E2E_LLM_MODEL;
const LLM_API = process.env.E2E_LLM_API ?? "openai-completions";
const THINKING_LEVELS = ["off", "minimal", "low", "medium", "high", "xhigh"];
const LLM_REASONING = (process.env.E2E_LLM_REASONING ?? "off").trim() || "off";
const KEEP_OV_SESSION = (process.env.E2E_KEEP_OV_SESSION ?? "") === "1";
const FAILCLOSED_MODE = (process.env.E2E_WINDOW_FAILCLOSED ?? "").trim();
const LONG_MODE = (process.env.E2E_WINDOW_LONG ?? "").trim();
const RUN_FAILCLOSED = FAILCLOSED_MODE === "1" || FAILCLOSED_MODE === "both";
const RUN_LONG = LONG_MODE === "1" || LONG_MODE === "both";
const RUN_MAIN = FAILCLOSED_MODE !== "1" && LONG_MODE !== "1";
/** Context share the long scenario must reach before the agent resets. */
const LONG_MIN_PERCENT = Number(process.env.E2E_WINDOW_LONG_MIN_PERCENT ?? 40);
/** Where the soft reminder fires: high enough that the run is a real workload. */
const LONG_SOFT_PERCENT = Number(process.env.E2E_WINDOW_LONG_SOFT_PERCENT ?? 45);
/** What models.json tells pi the window is; used to turn payload size into a share. */
const LONG_CONTEXT_WINDOW = 128000;
/** A port nothing listens on: the fail-closed variant's "OpenViking is down". */
const DEAD_OV_URL = "http://127.0.0.1:9";
const PROVIDER_ID = "e2e-relay";
const RESET_TOOL = "new_context";
const CODENAME = "ZEPHYR-9942";
const DEPLOY_WINDOW = "Friday 03:00 UTC";
const RELEASE_FILE = "release.md";
/** Only present in release.md, so answering with it proves the file was read. */
const RELEASE_OWNER = "dana-okonkwo";
const RESET_REASON = "phase one complete, starting phase two";
/** The part of the reason we require verbatim in the archive and the entry. */
const REASON_MARKER = "phase one complete";
const NEXT_STEP_MARKER = "rollout checklist";
const PAD1 = `PADDING-T1 ${"lorem ipsum dolor sit amet consectetur ".repeat(60)}`;
// ============================================================================
// Environment validation (runs before anything is created, so a bad env is a
// clean exit 2 and the script stays dry-runnable).
// ============================================================================
function which(cmd) {
const res = spawnSync("which", [cmd], { encoding: "utf8" });
return res.status === 0 ? res.stdout.trim() : "";
}
const PI_BIN = process.env.PI_BIN || which("pi");
for (const [key, value] of [
["OPENVIKING_URL", OV_URL],
["OPENVIKING_API_KEY", OV_KEY],
["E2E_LLM_API_KEY", LLM_KEY],
["E2E_LLM_BASE_URL", LLM_BASE],
["E2E_LLM_MODEL", LLM_MODEL],
["PI_BIN or pi on PATH", PI_BIN],
]) {
if (!value) {
console.error(`e2e-window: missing required ${key}`);
process.exit(2);
}
}
if (!existsSync(PI_BIN)) {
console.error(`e2e-window: pi binary not found: ${PI_BIN}`);
process.exit(2);
}
if (!THINKING_LEVELS.includes(LLM_REASONING)) {
console.error(
`e2e-window: E2E_LLM_REASONING must be one of ${THINKING_LEVELS.join(", ")}, got "${LLM_REASONING}"`,
);
process.exit(2);
}
if (LONG_MODE && !["1", "both"].includes(LONG_MODE)) {
console.error(`e2e-window: E2E_WINDOW_LONG must be "1" or "both", got "${LONG_MODE}"`);
process.exit(2);
}
if (FAILCLOSED_MODE && !["1", "both"].includes(FAILCLOSED_MODE)) {
console.error(`e2e-window: E2E_WINDOW_FAILCLOSED must be "1" or "both", got "${FAILCLOSED_MODE}"`);
process.exit(2);
}
// ============================================================================
// Reporting
// ============================================================================
let failures = 0;
let warnings = 0;
const pass = (msg) => console.log(` PASS ${msg}`);
const fail = (msg) => {
failures++;
console.error(` FAIL ${msg}`);
};
const warn = (msg) => {
warnings++;
console.log(` WARN ${msg}`);
};
const check = (cond, msg) => (cond ? pass(msg) : fail(msg));
const section = (title) => console.log(`\ne2e-window: --- ${title} ---`);
// ============================================================================
// Workspace
// ============================================================================
/**
* A private pi installation: its own agent dir (models.json + settings.json),
* its own copy of the extension, its own session dir and project cwd. Nothing
* touches the user's ~/.pi.
*/
function makeWorkspace(label, { windowConfig = {}, seedSourceTree = false } = {}) {
const root = mkdtempSync(join(tmpdir(), `ov-pi-window-${label}-`));
const ws = {
label,
root,
agentDir: join(root, "agent"),
// Deliberately outside the agent dir: pi is told to load it with `-e`, and
// a copy under <agentDir>/extensions could be picked up a second time.
extDir: join(root, "ext"),
outDir: join(root, "out"),
projDir: join(root, "proj"),
sessionDir: join(root, "sessions"),
};
for (const dir of [ws.agentDir, ws.extDir, ws.outDir, ws.projDir, ws.sessionDir]) {
mkdirSync(dir, { recursive: true });
}
writeFileSync(
join(ws.agentDir, "models.json"),
JSON.stringify(
{
providers: {
[PROVIDER_ID]: {
name: "OpenViking e2e relay",
baseUrl: LLM_BASE,
api: LLM_API,
apiKey: "$E2E_LLM_API_KEY",
...(LLM_API === "openai-completions"
? { authHeader: true, headers: { "x-session-id": "ov-pi-e2e-window" } }
: {}),
models: [
{
id: LLM_MODEL,
name: "OpenViking e2e model",
reasoning: LLM_REASONING !== "off",
// A custom relay is not recognised by pi-ai's URL-based
// auto-detection, so reasoning_effort has to be enabled here.
...(LLM_REASONING !== "off" && LLM_API === "openai-completions"
? { compat: { supportsReasoningEffort: true, thinkingFormat: "openai" } }
: {}),
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 8192,
},
],
},
},
},
null,
2,
),
);
writeFileSync(
join(ws.agentDir, "settings.json"),
JSON.stringify(
{
defaultProjectTrust: "always",
...(LLM_REASONING !== "off" ? { defaultThinkingLevel: LLM_REASONING } : {}),
},
null,
2,
),
);
// The extension itself: sources plus lib/shared/scripts, with a config that
// is generous about the archive deadline (phase 2 takes 25-55s on the target
// server) and eager about the reminders, so a short gate still exercises the
// thresholds.
for (const name of readdirSync(EXT_SRC)) {
if (name === "config.json") continue;
const src = join(EXT_SRC, name);
const dst = join(ws.extDir, basename(name));
if (name.endsWith(".ts") || name === "README.md") {
copyFileSync(src, dst);
} else if (["lib", "shared", "scripts"].includes(name)) {
cpSync(src, dst, { recursive: true });
}
}
writeFileSync(
join(ws.extDir, "config.json"),
JSON.stringify(
{
enabled: true,
syncTurns: true,
logLevel: "info",
contextWindow: {
resetDeadlineMs: 120000,
archivePollMs: 3000,
softPercent: 60,
hardPercent: 75,
statusEveryTurn: true,
...windowConfig,
},
},
null,
2,
),
);
// The long scenario needs enough real material that reading it through does
// not fit in one window. The extension's own sources are the closest thing to
// hand: about 60k tokens of TypeScript and ESM, already in the repo.
if (seedSourceTree) {
const srcDir = join(ws.projDir, "src");
mkdirSync(srcDir, { recursive: true });
let bytes = 0;
for (const name of readdirSync(EXT_SRC)) {
const from = join(EXT_SRC, name);
if (name.endsWith(".ts")) {
copyFileSync(from, join(srcDir, name));
bytes += statSync(from).size;
} else if (name === "lib" || name === "tests") {
for (const child of readdirSync(from)) {
if (!child.endsWith(".mjs")) continue;
copyFileSync(join(from, child), join(srcDir, child));
bytes += statSync(join(from, child)).size;
}
}
}
const files = readdirSync(srcDir).sort();
writeFileSync(
join(ws.projDir, "TASK.md"),
[
"# Source inventory task",
"",
`There are ${files.length} files under src/ (~${Math.round(bytes / 1024)} KB).`,
"Work through them in alphabetical order:",
"",
...files.map((f) => `- [ ] src/${f}`),
"",
].join("\n"),
);
console.log(
`e2e-window: [${label}] seeded ${files.length} source files, ` +
`${Math.round(bytes / 1024)} KB (~${Math.round(bytes / 4000)}k tokens if fully read)`,
);
}
// A file for T1 to read: the gate asserts that tool OUTPUT reaches the
// archive, so the first turn must produce a tool result rather than relying
// on the model deciding to use a tool on its own.
writeFileSync(
join(ws.projDir, RELEASE_FILE),
[
"# Release checklist",
"",
`- codename: ${CODENAME}`,
`- deploy window: ${DEPLOY_WINDOW}`,
`- owner: ${RELEASE_OWNER}`,
"- status: phase one in progress",
"",
].join("\n"),
);
console.log(`e2e-window: [${label}] workspace ${root}`);
return ws;
}
/** Run one pi turn in a workspace and capture stdout/stderr. */
function runTurn(ws, turn, prompt, { continueSession = false, ovUrl = OV_URL, timeoutMs = 600_000 } = {}) {
console.log(`\ne2e-window: [${ws.label}] --- turn ${turn} ---`);
const args = [
"--provider", PROVIDER_ID,
"--model", LLM_MODEL,
"--mode", "text",
"-p",
"-ne",
"-e", join(ws.extDir, "index.ts"),
"-e", join(ws.extDir, "scripts", "e2e-probe.ts"),
"--approve",
"--no-context-files",
"--no-skills",
"--no-prompt-templates",
"--session-dir", ws.sessionDir,
"--offline",
...(continueSession ? ["-c"] : []),
prompt,
];
const res = spawnSync(PI_BIN, args, {
cwd: ws.projDir,
env: {
...process.env,
PI_CODING_AGENT_DIR: ws.agentDir,
OPENVIKING_URL: ovUrl,
OPENVIKING_API_KEY: OV_KEY,
E2E_LLM_API_KEY: LLM_KEY,
OV_E2E_OUT: ws.outDir,
OV_E2E_TURN: String(turn),
OPENVIKING_DEBUG_LOG: join(ws.outDir, "ov-pi.log"),
},
timeout: timeoutMs,
encoding: "utf8",
});
const out = `${res.stdout ?? ""}`;
const err = `${res.stderr ?? ""}`;
if (res.status !== 0) {
fail(
`[${ws.label}] turn ${turn}: pi exited ${res.status}\n` +
`stdout:\n${out.slice(-2000)}\nstderr:\n${err.slice(-2000)}`,
);
} else {
console.log(out.trim().slice(-800));
}
return { out, err, status: res.status, transcript: `${out}\n${err}` };
}
// ============================================================================
// OpenViking HTTP
// ============================================================================
async function ovFetch(path, init, baseUrl = OV_URL) {
try {
const resp = await fetch(`${baseUrl}${path}`, {
...init,
headers: {
Authorization: `Bearer ${OV_KEY}`,
"Content-Type": "application/json",
...(init?.headers ?? {}),
},
});
const body = await resp.json().catch(() => ({}));
return { ok: resp.ok && body?.status !== "error", status: resp.status, body };
} catch (error) {
return { ok: false, status: 0, body: {}, error: String(error?.message ?? error) };
}
}
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
/** `viking://…/sessions/<sid>` — canonical root, legacy alias as the fallback. */
async function sessionRootUri(ovSessionId) {
const res = await ovFetch(`/api/v1/sessions/${encodeURIComponent(ovSessionId)}`);
const uri = typeof res.body?.result?.uri === "string" ? res.body.result.uri.trim() : "";
if (uri.startsWith("viking://")) return uri.replace(/\/+$/, "");
return `viking://~/sessions/${ovSessionId}`;
}
/** Archive directories under `<root>/history`, newest first. */
async function listArchives(rootUri) {
const res = await ovFetch(
`/api/v1/fs/ls?uri=${encodeURIComponent(`${rootUri}/history`)}&sort_by=name&sort_order=desc`,
);
const rows = Array.isArray(res.body?.result) ? res.body.result : [];
const archives = [];
for (const row of rows) {
const uri = String(row?.uri ?? "").replace(/\/+$/, "");
const id = uri.split("/").pop() || String(row?.name ?? "");
if (!/^archive_\d+$/.test(id)) continue;
archives.push({ archiveId: id, uri: uri || `${rootUri}/history/${id}` });
}
archives.sort((a, b) => Number(b.archiveId.slice(8)) - Number(a.archiveId.slice(8)));
return archives;
}
/** `GET /content/read?uri=<archive>/<file>` — raw text, plus the HTTP status. */
async function readArchiveFile(archiveUri, file) {
const res = await ovFetch(
`/api/v1/content/read?uri=${encodeURIComponent(`${archiveUri}/${file}`)}`,
);
return { status: res.status, ok: res.ok, text: typeof res.body?.result === "string" ? res.body.result : "" };
}
/** Poll `.overview.md` until phase 2 has written it (25-55s on the target server). */
async function waitForOverview(archiveUri, budgetMs = 150_000) {
const startedAt = Date.now();
let last = { status: 0, ok: false, text: "" };
while (Date.now() - startedAt < budgetMs) {
last = await readArchiveFile(archiveUri, ".overview.md");
if (last.status === 200 && last.text.trim()) break;
await sleep(5000);
}
return { ...last, waitedMs: Date.now() - startedAt };
}
// ============================================================================
// Provider payload helpers
// ============================================================================
function payloadsFor(ws, turn) {
return readdirSync(ws.outDir)
.filter((f) => f.startsWith(`payload-t${turn}-`) && f.endsWith(".json"))
.sort()
.map((f) => {
const raw = readFileSync(join(ws.outDir, f), "utf8");
let payload = null;
try {
payload = JSON.parse(raw);
} catch {
payload = null;
}
return { file: f, raw, payload };
});
}
const asArray = (value) => (Array.isArray(value) ? value : []);
const contentBlocks = (msg) => (Array.isArray(msg?.content) ? msg.content : []);
/** Text of a message under either provider shape. */
function messageText(msg) {
if (typeof msg?.content === "string") return msg.content;
return contentBlocks(msg)
.filter((b) => b && (b.type === "text" || b.type === "input_text") && typeof b.text === "string")
.map((b) => b.text)
.join("\n");
}
function messagesOf(entry) {
return asArray(entry?.payload?.messages);
}
/** The first message the model actually reads (system/developer are framing). */
function firstNonSystem(messages) {
return messages.find((m) => m?.role !== "system" && m?.role !== "developer") ?? null;
}
/** `{id, name}` for every tool call in a message, both provider shapes. */
function toolCallsOf(msg) {
const calls = [];
for (const call of asArray(msg?.tool_calls)) {
calls.push({ id: String(call?.id ?? ""), name: String(call?.function?.name ?? call?.name ?? "") });
}
for (const block of contentBlocks(msg)) {
if (block?.type === "tool_use") calls.push({ id: String(block?.id ?? ""), name: String(block?.name ?? "") });
}
return calls;
}
/** `{id}` for every tool result in a message, both provider shapes. */
function toolResultsOf(msg) {
const results = [];
// openai-completions: a dedicated {role:"tool", tool_call_id} message.
if (msg?.role === "tool") results.push({ id: String(msg?.tool_call_id ?? "") });
// anthropic-messages: tool_result blocks carried on a user message.
for (const block of contentBlocks(msg)) {
if (block?.type === "tool_result") results.push({ id: String(block?.tool_use_id ?? "") });
}
return results;
}
/**
* Orphan detection for both wire shapes.
*
* A tool result whose call was cut away — or a call whose result was cut away —
* is rejected outright by the provider, and it is exactly what a sloppy cut
* produces, so it is the single most important structural assertion here.
*/
function assertNoOrphanToolResults(messages, label) {
const issued = new Set();
const answered = new Set();
const orphanResults = [];
for (const msg of messages) {
for (const call of toolCallsOf(msg)) {
if (call.id) issued.add(call.id);
}
for (const result of toolResultsOf(msg)) {
if (!result.id || !issued.has(result.id)) {
orphanResults.push(result.id || "(missing id)");
} else {
answered.add(result.id);
}
}
}
const danglingCalls = [...issued].filter((id) => !answered.has(id));
check(
orphanResults.length === 0,
`${label}: no orphan tool results${orphanResults.length ? ` (orphans: ${orphanResults.join(", ")})` : ""}`,
);
check(
danglingCalls.length === 0,
`${label}: no tool calls left without a result${danglingCalls.length ? ` (dangling: ${danglingCalls.join(", ")})` : ""}`,
);
return { orphanResults, danglingCalls };
}
/** Does any non-system message call `new_context` (or answer such a call)? */
function mentionsResetToolCall(messages) {
const resetIds = new Set();
let found = false;
for (const msg of messages) {
for (const call of toolCallsOf(msg)) {
if (call.name === RESET_TOOL) {
found = true;
if (call.id) resetIds.add(call.id);
}
}
for (const result of toolResultsOf(msg)) {
if (result.id && resetIds.has(result.id)) found = true;
}
}
return found;
}
function lastAssistant(messages) {
for (let i = messages.length - 1; i >= 0; i--) {
if (messages[i]?.role === "assistant") return messages[i];
}
return null;
}
// ============================================================================
// Session file helpers
// ============================================================================
function readSessionEntries(ws) {
const files = readdirSync(ws.sessionDir)
.filter((f) => f.endsWith(".jsonl"))
.sort();
if (files.length === 0) return { file: "", entries: [] };
const file = join(ws.sessionDir, files[files.length - 1]);
const entries = [];
for (const line of readFileSync(file, "utf8").split("\n")) {
const trimmed = line.trim();
if (!trimmed) continue;
try {
entries.push(JSON.parse(trimmed));
} catch {
// A partially flushed tail must not lose the entries before it.
}
}
return { file, entries };
}
// ============================================================================
// Prompts
// ============================================================================
const T1_PROMPT =
`Read the file ${RELEASE_FILE} in the working directory with the read tool, then tell me who owns ` +
`the release. Also remember for the rest of this session: the release codename is ${CODENAME}, ` +
`and the deploy window is ${DEPLOY_WINDOW}. Keep the answer to one short sentence. ${PAD1}`;
const T2_PROMPT =
"Phase one is complete. Do exactly this, nothing else. " +
`Make a single tool call to ${RESET_TOOL}, alone in its own turn, with no other tool call beside it. ` +
`Use reason exactly: "${RESET_REASON}". ` +
`In notes, state the release codename ${CODENAME}, the deploy window ${DEPLOY_WINDOW}, ` +
`and that the next step is to write the phase two ${NEXT_STEP_MARKER}. ` +
"After the new context window opens, reply with exactly: READY";
const T3_PROMPT = "What is the release codename I gave you earlier? Answer with just the codename.";
/**
* The long scenario never mentions the context tools: the point is that the
* agent reaches for them on its own once the window fills up. The work is
* deliberately bigger than one window, and the running file means progress is
* visible even across a reset.
*/
const LONG_T1_PROMPT =
"Read TASK.md, then work through every file it lists, in order. For each file: read it with the " +
"read tool, and append one entry to INVENTORY.md with the file name, what it is responsible for, " +
"its main exports, and anything that looks risky. Tick the file off in TASK.md as you go. " +
"Append after each file rather than holding everything until the end, and keep going until every " +
"file is covered. Do not summarise the files you have not read yet.";
const LONG_T2_PROMPT =
"Continue the inventory exactly where you left off: read TASK.md and INVENTORY.md first to see " +
"what is already done, then carry on until every file is ticked off.";
const LONG_T3_PROMPT =
"In the very first file you inventoried, what did you write down as its main responsibility, and " +
"what was the first thing that file imports? Answer in two short lines.";
// ============================================================================
// Scenario: the main gate
// ============================================================================
async function runMainScenario() {
const ws = makeWorkspace("main");
const t1 = runTurn(ws, 1, T1_PROMPT);
const t2 = runTurn(ws, 2, T2_PROMPT, { continueSession: true });
const t3 = runTurn(ws, 3, T3_PROMPT, { continueSession: true });
section("pi runs");
check(
t1.status === 0 && t2.status === 0 && t3.status === 0,
`all three pi runs exited 0 (got ${t1.status}/${t2.status}/${t3.status})`,
);
// ---- T2 payloads: the cut ------------------------------------------------
section("T2 provider payloads");
const p2 = payloadsFor(ws, 2);
check(p2.length >= 2, `T2 produced >=2 provider payloads (got ${p2.length})`);
const postIndex = p2.findIndex((entry) => {
const first = firstNonSystem(messagesOf(entry));
return first?.role === "user" && messageText(first).startsWith(WINDOW_HEADER_OPEN);
});
check(postIndex >= 0, "T2 has a post-cut payload that starts with the window header");
if (postIndex > 0) {
const pre = p2[postIndex - 1];
const post = p2[postIndex];
const preMessages = messagesOf(pre);
const postMessages = messagesOf(post);
const header = messageText(firstNonSystem(postMessages));
const preCall = lastAssistant(preMessages);
if (preCall && toolCallsOf(preCall).some((c) => c.name === RESET_TOOL)) {
pass(`pre-cut payload (${pre.file}) ends with an assistant ${RESET_TOOL} tool call`);
} else {
// The happy path is one sampling per turn: the model emits the
// new_context call as its very first response, so the request that
// elicited it cannot contain it yet. That is correct behaviour, not a
// regression — but the pre-cut payload must still be the *old* window.
warn(
`pre-cut payload (${pre.file}) has no assistant ${RESET_TOOL} call: the model called it on ` +
"its first sampling of the turn, so no captured request contains the call",
);
check(pre.raw.includes(PAD1), "pre-cut payload is the uncut window (still carries PAD1)");
check(
!messageText(firstNonSystem(preMessages)).startsWith(WINDOW_HEADER_OPEN),
"pre-cut payload does not already start with a window header",
);
}
check(header.includes('id="w2"'), 'window header is the second window (id="w2")');
check(header.includes(CODENAME), `window header carries the codename ${CODENAME} from the notes`);
check(!post.raw.includes(PAD1), "post-cut payload no longer contains the T1 padding");
check(
postMessages.length < preMessages.length,
`post-cut payload has fewer messages (${postMessages.length} < ${preMessages.length})`,
);
assertNoOrphanToolResults(preMessages, "pre-cut payload");
assertNoOrphanToolResults(postMessages, "post-cut payload");
check(
!mentionsResetToolCall(postMessages),
`post-cut payload contains neither the ${RESET_TOOL} call nor its result`,
);
} else if (postIndex === 0) {
fail("the very first T2 payload already carries the window header; no cut was observed in T2");
}
// ---- T3 payloads: the window survived the process boundary ---------------
section("T3 provider payloads");
const p3 = payloadsFor(ws, 3);
check(p3.length > 0, `T3 produced a provider payload (got ${p3.length})`);
if (p3.length > 0) {
const last = p3[p3.length - 1];
const first = firstNonSystem(messagesOf(last));
check(
first?.role === "user" && messageText(first).startsWith(WINDOW_HEADER_OPEN),
"T3 payload still starts with the frozen window header (restored in a new process)",
);
check(!last.raw.includes(PAD1), "T3 payload still does not contain the T1 padding");
}
if (t3.out.includes(CODENAME)) pass(`T3 answer contains ${CODENAME}`);
else warn(`T3 answer did not contain ${CODENAME}; inspect the window header and Working Memory`);
// ---- Session file: window entry + status line ----------------------------
section("pi session file");
const { file: sessionFile, entries } = readSessionEntries(ws);
check(Boolean(sessionFile), `session JSONL found (${sessionFile || "none"})`);
const windowEntries = entries.filter(
(e) => e?.type === "custom" && e?.customType === WINDOW_ENTRY_TYPE && e?.data,
);
const armedEntries = windowEntries.filter((e) => String(e.data?.archiveId ?? ""));
const distinctWindows = new Set(armedEntries.map((e) => `${e.data.windowIndex}:${e.data.archiveId}`));
check(
distinctWindows.size === 1,
`exactly one ${WINDOW_ENTRY_TYPE} window recorded (distinct windows: ${distinctWindows.size}, ` +
`raw entries: ${windowEntries.length})`,
);
if (windowEntries.length > armedEntries.length + 1 || armedEntries.length > 1) {
// Every persist() that sees a changed watermark writes another entry; only
// a *second window* is a defect.
warn(
`${windowEntries.length} ${WINDOW_ENTRY_TYPE} entries in the session file ` +
`(${armedEntries.length} armed): re-persists of the same window are expected`,
);
}
const windowState = armedEntries.length > 0 ? armedEntries[armedEntries.length - 1].data : null;
check(
Boolean(windowState) && String(windowState.reason ?? "").includes(REASON_MARKER),
`window entry records the reason (got "${String(windowState?.reason ?? "").slice(0, 80)}")`,
);
check(
Boolean(windowState) && /^archive_\d+$/.test(String(windowState.archiveId ?? "")),
`window entry records an archive id (got "${windowState?.archiveId ?? ""}")`,
);
const statusEntries = entries.filter(
(e) =>
(e?.type === "custom_message" || e?.type === "custom") && e?.customType === STATUS_CUSTOM_TYPE,
);
check(statusEntries.length >= 1, `session file has >=1 ${STATUS_CUSTOM_TYPE} message (got ${statusEntries.length})`);
// ---- Debug log -----------------------------------------------------------
section("debug log");
const logPath = join(ws.outDir, "ov-pi.log");
if (existsSync(logPath)) {
const log = readFileSync(logPath, "utf8");
check(log.includes("context-window: opened w2"), 'debug log records "context-window: opened w2"');
} else {
fail(`no debug log at ${logPath}`);
}
// ---- OpenViking ----------------------------------------------------------
section("OpenViking archive");
const sessionIdFile = join(ws.outDir, "session-id.txt");
let ovSessionId = null;
if (!existsSync(sessionIdFile)) {
fail("probe did not record a pi session id");
} else {
ovSessionId = `pi-${readFileSync(sessionIdFile, "utf8").trim()}`;
const rootUri = await sessionRootUri(ovSessionId);
console.log(`e2e-window: OV session ${ovSessionId} root ${rootUri}`);
const archives = await listArchives(rootUri);
check(archives.length >= 1, `session has >=1 archive under <root>/history (got ${archives.length})`);
if (archives.length > 0) {
const archive = archives[0];
if (windowState?.archiveId) {
check(
archive.archiveId === windowState.archiveId,
`newest archive matches the window entry (${archive.archiveId} vs ${windowState.archiveId})`,
);
}
const overview = await waitForOverview(archive.uri);
check(
overview.status === 200 && overview.text.trim().length > 0,
`GET content/read ${archive.archiveId}/.overview.md is 200 with a body ` +
`(status ${overview.status}, waited ${overview.waitedMs}ms)`,
);
const messages = await readArchiveFile(archive.uri, "messages.jsonl");
check(messages.status === 200 && messages.text.length > 0, `${archive.archiveId}/messages.jsonl is readable`);
check(messages.text.includes(HANDOFF_MARKER), `messages.jsonl contains "${HANDOFF_MARKER}"`);
check(messages.text.includes(REASON_MARKER), `messages.jsonl contains the reason "${REASON_MARKER}"`);
check(messages.text.includes("[tool-result"), 'messages.jsonl contains a "[tool-result" entry (tool output reached the archive)');
const grep = await ovFetch("/api/v1/search/grep", {
method: "POST",
body: JSON.stringify({
uri: `${rootUri}/history`,
pattern: "ZEPHYR",
case_insensitive: true,
}),
});
const matches = asArray(grep.body?.result?.matches);
check(matches.length >= 1, `grep "ZEPHYR" over <root>/history has >=1 match (got ${matches.length})`);
}
if (KEEP_OV_SESSION) {
console.log(
`e2e-window: keeping OV session ${ovSessionId} (E2E_KEEP_OV_SESSION=1); ` +
`archives stay under <session root>/history — delete it yourself when done`,
);
} else {
const del = await ovFetch(`/api/v1/sessions/${encodeURIComponent(ovSessionId)}`, { method: "DELETE" });
console.log(
`e2e-window: cleanup OV session ${ovSessionId}: ${del.ok ? "deleted" : "FAILED; delete it manually"}`,
);
}
}
return ws;
}
// ============================================================================
// Scenario: fail-closed
// ============================================================================
/**
* Highest context share the extension's own status line reported, and the
* biggest provider request seen. Both are evidence that the window really did
* fill up before the agent reset it.
*/
function contextPressureSeen(ws, entries) {
let peakPercent = 0;
let peakTokens = "";
for (const entry of entries) {
if (entry?.customType !== STATUS_CUSTOM_TYPE) continue;
const text = typeof entry.content === "string" ? entry.content : "";
const match = text.match(/~([\d.]+k?)\/(\S+)\s+tokens\s+\((\d+)%\)/);
if (!match) continue;
const percent = Number(match[3]);
if (percent > peakPercent) {
peakPercent = percent;
peakTokens = `${match[1]}/${match[2]}`;
}
}
let peakBytes = 0;
let peakPayload = "";
for (const turn of [1, 2, 3]) {
for (const entry of payloadsFor(ws, turn)) {
const bytes = messagesOf(entry).reduce((sum, m) => sum + JSON.stringify(m).length, 0);
if (bytes > peakBytes) {
peakBytes = bytes;
peakPayload = entry.file;
}
}
}
// chars/4 is the same estimate the extension itself uses for non-CJK text.
const peakEstimatedTokens = Math.round(peakBytes / 4);
const peakEstimatedPercent = Math.round((peakEstimatedTokens / LONG_CONTEXT_WINDOW) * 100);
return {
peakPercent,
peakTokens,
peakBytes,
peakPayload,
peakEstimatedTokens,
peakEstimatedPercent,
};
}
/**
* The OpenViking side of a scenario that does not control when the reset
* happens: assert the archive exists and carries the handoff, without assuming
* a particular reason string.
*/
async function checkOpenVikingSide(ws, { requireHandoff }) {
section("long: OpenViking archive");
const sessionIdFile = join(ws.outDir, "session-id.txt");
if (!existsSync(sessionIdFile)) {
fail("probe did not record a pi session id");
return;
}
const ovSessionId = `pi-${readFileSync(sessionIdFile, "utf8").trim()}`;
const rootUri = await sessionRootUri(ovSessionId);
console.log(`e2e-window: OV session ${ovSessionId} root ${rootUri}`);
const archives = await listArchives(rootUri);
if (!requireHandoff && archives.length === 0) {
warn("no archive: the agent never reset, so nothing was committed");
} else {
check(archives.length >= 1, `session has >=1 archive under <root>/history (got ${archives.length})`);
}
if (archives.length > 0) {
const newest = archives[0];
const overview = await waitForOverview(newest.uri);
check(
overview.status === 200 && overview.text.trim().length > 0,
`${newest.archiveId}/.overview.md is readable, Working Memory generated ` +
`(status ${overview.status}, waited ${overview.waitedMs}ms)`,
);
const messages = await readArchiveFile(newest.uri, "messages.jsonl");
check(
messages.status === 200 && messages.text.length > 0,
`${newest.archiveId}/messages.jsonl is readable`,
);
if (messages.text) {
check(messages.text.includes(HANDOFF_MARKER), `messages.jsonl contains "${HANDOFF_MARKER}"`);
check(
messages.text.includes("[tool-result"),
'messages.jsonl contains a "[tool-result" entry (tool output reached the archive)',
);
}
}
if (KEEP_OV_SESSION) {
console.log(
`e2e-window: keeping OV session ${ovSessionId} (E2E_KEEP_OV_SESSION=1); ` +
"delete it yourself when done",
);
} else {
const del = await ovFetch(`/api/v1/sessions/${encodeURIComponent(ovSessionId)}`, { method: "DELETE" });
console.log(
`e2e-window: cleanup OV session ${ovSessionId}: ${del.ok ? "deleted" : "FAILED; delete it manually"}`,
);
}
}
// ============================================================================
// Scenario: long context, the agent resets under pressure on its own
// ============================================================================
async function runLongScenario() {
const ws = makeWorkspace("long", {
seedSourceTree: true,
// Soft reminder well before pi's own compaction line, so the agent has room
// to finish the file it is on, write notes and reset deliberately.
windowConfig: { softPercent: LONG_SOFT_PERCENT, hardPercent: 70 },
});
// Reading 400 KB of source with reasoning on is slow: a turn that gets killed
// mid-inventory looks like a product failure in the transcript when it is
// only the harness giving up.
const turnTimeoutMs = Number(process.env.E2E_WINDOW_LONG_TURN_TIMEOUT_MS ?? 1_500_000);
const t1 = runTurn(ws, 1, LONG_T1_PROMPT, { timeoutMs: turnTimeoutMs });
const t2 = runTurn(ws, 2, LONG_T2_PROMPT, { continueSession: true, timeoutMs: turnTimeoutMs });
const t3 = runTurn(ws, 3, LONG_T3_PROMPT, { continueSession: true, timeoutMs: turnTimeoutMs });
section("long: pi runs");
check(
t1.status === 0 && t2.status === 0 && t3.status === 0,
`all three pi runs exited 0 (got ${t1.status}/${t2.status}/${t3.status})`,
);
const { file: sessionFile, entries } = readSessionEntries(ws);
check(Boolean(sessionFile), "session JSONL found");
section("long: context pressure");
const pressure = contextPressureSeen(ws, entries);
const { peakPercent, peakTokens, peakBytes, peakPayload, peakEstimatedTokens, peakEstimatedPercent } =
pressure;
console.log(
`e2e-window: biggest provider request ${peakPayload} ${Math.round(peakBytes / 1024)} KB ` +
`(~${Math.round(peakEstimatedTokens / 1000)}k tokens, ${peakEstimatedPercent}% of ` +
`${Math.round(LONG_CONTEXT_WINDOW / 1000)}k)`,
);
console.log(
`e2e-window: highest [context-status] line seen: ${peakPercent}% (${peakTokens || "none"}) — ` +
"that line is emitted once per user prompt, so it undersamples a long tool-heavy turn",
);
// The payload is what actually went to the model; the status line is a sample.
const reached = Math.max(peakPercent, peakEstimatedPercent);
if (reached >= LONG_MIN_PERCENT) {
pass(`context reached ${reached}% of the window (>= ${LONG_MIN_PERCENT}%)`);
} else {
warn(
`context only reached ${reached}% (< ${LONG_MIN_PERCENT}%): the model stopped reading ` +
"early, so this run does not exercise the pressure path",
);
}
section("long: the agent's own decision");
const resets = [];
for (const entry of entries) {
for (const msg of [entry?.message].filter(Boolean)) {
for (const block of asArray(msg.content)) {
if (block?.type === "toolCall" && block?.name === RESET_TOOL) {
resets.push(block.arguments ?? {});
}
}
}
}
const windowEntries = entries.filter((e) => e?.customType === WINDOW_ENTRY_TYPE);
const armed = windowEntries.filter((e) => e?.data?.anchorToolCallId);
if (resets.length === 0) {
warn("the agent never called new_context: nothing was instructed, so this is model judgement");
} else {
pass(`the agent called ${RESET_TOOL} ${resets.length}x without being told to`);
console.log(`e2e-window: reason: ${JSON.stringify(resets[0]?.reason ?? "")}`);
console.log(`e2e-window: notes: ${JSON.stringify(String(resets[0]?.notes ?? "").slice(0, 300))}`);
check(
String(resets[0]?.notes ?? "").length > 40,
"the reset carried handoff notes of its own",
);
check(armed.length > 0, `a window was armed and persisted (${armed.length} entries)`);
}
section("long: the cut");
if (resets.length === 0) {
warn("no reset, so there is no cut to check");
} else {
const afterCut = [];
for (const turn of [1, 2, 3]) {
for (const entry of payloadsFor(ws, turn)) {
const first = firstNonSystem(messagesOf(entry));
if (first && messageText(first).startsWith(WINDOW_HEADER_OPEN)) afterCut.push(entry);
}
}
check(afterCut.length > 0, `at least one provider request starts with the window header (${afterCut.length})`);
for (const entry of afterCut.slice(0, 2)) {
assertNoOrphanToolResults(messagesOf(entry), `post-cut payload ${entry.file}`);
}
const post = afterCut[0];
if (post) {
const postBytes = messagesOf(post).reduce((sum, m) => sum + JSON.stringify(m).length, 0);
check(
postBytes * 2 < peakBytes,
`the reset more than halved the request (${Math.round(peakBytes / 1024)} KB -> ` +
`${Math.round(postBytes / 1024)} KB)`,
);
check(
!mentionsResetToolCall(messagesOf(post)),
"the reset tool call and its result are gone from the new window",
);
}
}
section("long: work continued across the boundary");
const inventory = join(ws.projDir, "INVENTORY.md");
const taskFile = join(ws.projDir, "TASK.md");
if (existsSync(inventory)) {
const body = readFileSync(inventory, "utf8");
const sourceFiles = readdirSync(join(ws.projDir, "src")).sort();
const covered = sourceFiles.filter((f) => body.includes(f));
console.log(`e2e-window: INVENTORY.md covers ${covered.length}/${sourceFiles.length} files`);
check(covered.length > 0, "the agent produced an inventory");
if (resets.length > 0 && covered.length <= 1) {
warn("only one file made it into the inventory: the run stopped too early to show continuity");
}
if (existsSync(taskFile)) {
const ticked = (readFileSync(taskFile, "utf8").match(/- \[x\]/gi) ?? []).length;
console.log(`e2e-window: TASK.md ticked off ${ticked} files`);
}
} else {
warn("no INVENTORY.md: the agent never started the task");
}
await checkOpenVikingSide(ws, { requireHandoff: resets.length > 0 });
return ws;
}
async function runFailClosedScenario() {
const ws = makeWorkspace("failclosed");
console.log(`e2e-window: [failclosed] OPENVIKING_URL=${DEAD_OV_URL}`);
const f1 = runTurn(ws, 1, T1_PROMPT, { ovUrl: DEAD_OV_URL });
const f2 = runTurn(ws, 2, T2_PROMPT, { continueSession: true, ovUrl: DEAD_OV_URL });
section("fail-closed");
check(f1.status === 0 && f2.status === 0, `both fail-closed pi runs exited 0 (got ${f1.status}/${f2.status})`);
const payloads = payloadsFor(ws, 2);
check(payloads.length >= 1, `fail-closed T2 produced a provider payload (got ${payloads.length})`);
if (payloads.length > 0) {
const last = payloads[payloads.length - 1];
check(last.raw.includes(PAD1), "fail-closed: the last payload still contains PAD1 (nothing was cut)");
const first = firstNonSystem(messagesOf(last));
check(
!messageText(first).startsWith(WINDOW_HEADER_OPEN),
"fail-closed: no window header was injected",
);
assertNoOrphanToolResults(messagesOf(last), "fail-closed last payload");
}
const logPath = join(ws.outDir, "ov-pi.log");
const haystack = [
f2.transcript,
...payloads.map((p) => p.raw),
existsSync(logPath) ? readFileSync(logPath, "utf8") : "",
].join("\n");
check(haystack.includes("NOT reset"), 'fail-closed: the refusal text "NOT reset" reached the model');
return ws;
}
// ============================================================================
// Main
// ============================================================================
const workspaces = [];
try {
if (RUN_MAIN) workspaces.push(await runMainScenario());
if (RUN_LONG) workspaces.push(await runLongScenario());
if (RUN_FAILCLOSED) workspaces.push(await runFailClosedScenario());
} catch (error) {
fail(`unexpected error: ${error?.stack || error}`);
}
console.log("");
for (const ws of workspaces) {
if (failures === 0 && !process.env.E2E_KEEP_TMP) {
rmSync(ws.root, { recursive: true, force: true });
} else {
console.log(`e2e-window: [${ws.label}] workspace kept for inspection: ${ws.root}`);
}
}
console.log(
failures === 0
? `\ne2e-window: ALL CHECKS PASSED${warnings ? ` (${warnings} warning(s))` : ""}`
: `\ne2e-window: ${failures} CHECK(S) FAILED${warnings ? `, ${warnings} warning(s)` : ""}`,
);
process.exit(failures === 0 ? 0 : 1);