Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
export const TAKEOVER_ENTRY_TYPE = "ov-takeover";
export const OVERVIEW_MARKER = "[OpenViking Session Context]";
const DEFAULT_CONFIG = {
takeoverEnabled: false,
takeoverTokenThreshold: 30000,
takeoverKeepRecentTurns: 3,
takeoverOverviewBudget: 3000,
takeoverOverviewPollMs: 2000,
takeoverOverviewPollMax: 15,
};
// pi hosts cap every extension event handler at 30s; omp logs `handler timed
// out after 30000ms`, discards the result and lets the handler run on (#5275).
// Takeover's work in one handler stays inside this budget.
export const HANDLER_BUDGET_MS = 25_000;
// What a commit plus one overview read needs; with less left, the commit waits
// for a later turn instead of running past the host's cap.
const COMMIT_RESERVE_MS = 10_000;
// One overview read (the client caps it at 5s) with room to spare.
const READ_RESERVE_MS = 5_000;
// Archive states after which its summary will not appear any more: the
// server's `.done` / `.failed.json` markers (idea from #5320).
const TERMINAL_ARCHIVE_STATES = new Set(["completed", "failed"]);
function numberOr(value, fallback) {
const next = Number(value);
return Number.isFinite(next) ? next : fallback;
}
function takeoverConfig(config = {}) {
return {
takeoverEnabled: config.takeoverEnabled === true,
takeoverTokenThreshold: Math.max(0, numberOr(config.takeoverTokenThreshold, DEFAULT_CONFIG.takeoverTokenThreshold)),
takeoverKeepRecentTurns: Math.max(0, numberOr(config.takeoverKeepRecentTurns, DEFAULT_CONFIG.takeoverKeepRecentTurns)),
takeoverOverviewBudget: Math.max(1, numberOr(config.takeoverOverviewBudget, DEFAULT_CONFIG.takeoverOverviewBudget)),
takeoverOverviewPollMs: Math.max(0, numberOr(config.takeoverOverviewPollMs, DEFAULT_CONFIG.takeoverOverviewPollMs)),
takeoverOverviewPollMax: Math.max(1, numberOr(config.takeoverOverviewPollMax, DEFAULT_CONFIG.takeoverOverviewPollMax)),
};
}
function asEntry(value) {
if (!value || typeof value !== "object") return null;
return value.entry && typeof value.entry === "object" ? value.entry : value;
}
function isUserEntry(entry) {
return entry?.type === "message" && isUserTurnStart(entry.message);
}
/** The role of the non-system message an entry puts into the context, or "". */
function contextRole(entry) {
if (entry?.type === "message") {
const role = entry.message?.role;
return typeof role === "string" && role !== "system" ? role : "";
}
if (entry?.type === "custom_message") return "custom";
if (entry?.type === "branch_summary" && entry.summary) return "branchSummary";
return "";
}
/**
* The branch entries pi builds the model context from, in context order —
* pi's own `buildContextEntries` projection, reimplemented because pi < 0.86
* does not expose it. After the latest compaction only the compaction entry,
* the entries from its `firstKeptEntryId` (system messages excluded) and
* everything after it remain; entries a context edit removed are dropped.
* `getBranch()` itself still holds the compacted-away prefix and every custom
* entry, so turn positions must never be counted on it directly.
*/
export function projectContextEntries(branch) {
const path = (Array.isArray(branch) ? branch : []).filter((entry) => entry && typeof entry === "object");
let compactionIdx = -1;
for (let i = path.length - 1; i >= 0; i--) {
if (path[i].type === "compaction") {
compactionIdx = i;
break;
}
}
let entries = path;
if (compactionIdx >= 0) {
const compaction = path[compactionIdx];
const keptIdx = path.findIndex((entry, i) => i < compactionIdx && entry.id === compaction.firstKeptEntryId);
const kept = keptIdx >= 0
? path.slice(keptIdx, compactionIdx).filter((entry) => !(entry.type === "message" && entry.message?.role === "system"))
: [];
entries = [compaction, ...kept, ...path.slice(compactionIdx + 1)];
}
const removed = new Set();
for (const entry of entries) {
if (entry.type === "context_edit" && entry.replacement === null && typeof entry.targetId === "string") {
removed.add(entry.targetId);
}
}
return removed.size ? entries.filter((entry) => !removed.has(entry.id)) : entries;
}
/**
* Position of a user entry's message among the `context` hook's messages.
* Matched on pi's per-message timestamp, which context edits and other
* extensions' content rewrites leave alone; the content fingerprint stands in
* only when a timestamp is missing. When several messages match, the one at
* the expected user-turn ordinal wins.
*/
function findUserMessage(messages, message, ordinal) {
const ts = message?.timestamp;
const byTime = typeof ts === "number" && Number.isFinite(ts);
const fp = byTime ? "" : fingerprintMessage(message);
let first = -1;
let seen = 0;
for (let i = 0; i < messages.length; i++) {
if (!isUserTurnStart(messages[i])) continue;
const match = byTime ? messages[i].timestamp === ts : fingerprintMessage(messages[i]) === fp;
if (match) {
if (seen === ordinal) return i;
if (first < 0) first = i;
}
seen++;
}
return first;
}
function currentBranch(branchOrGetter) {
if (typeof branchOrGetter === "function") {
try {
const value = branchOrGetter();
return Array.isArray(value) ? value : [];
} catch {
return [];
}
}
return Array.isArray(branchOrGetter) ? branchOrGetter : [];
}
function flattenValue(value) {
if (typeof value === "string") return value;
if (Array.isArray(value)) return value.map(flattenValue).filter(Boolean).join("");
if (!value || typeof value !== "object") return "";
if (typeof value.text === "string") return value.text;
if (typeof value.input_text === "string") return value.input_text;
if (typeof value.output_text === "string") return value.output_text;
if (typeof value.content === "string") return value.content;
if (Array.isArray(value.content)) return flattenValue(value.content);
return "";
}
export function flattenContent(msg) {
if (!msg || typeof msg !== "object") return "";
return flattenValue(msg.content);
}
export function fingerprintMessage(msg) {
const text = flattenContent(msg);
return `${msg?.role || ""}:${text.length}:${text.slice(0, 200)}`;
}
export function isUserTurnStart(msg) {
if (!msg || msg.role !== "user") return false;
return !flattenContent(msg).startsWith(OVERVIEW_MARKER);
}
export function countUserTurns(messages) {
let count = 0;
for (const msg of Array.isArray(messages) ? messages : []) {
if (isUserTurnStart(msg)) count++;
}
return count;
}
export function findBoundaryIndex(messages, coveredUserTurns) {
const target = Math.max(0, Math.floor(Number(coveredUserTurns) || 0)) + 1;
let seen = 0;
for (let i = 0; i < (Array.isArray(messages) ? messages.length : 0); i++) {
if (!isUserTurnStart(messages[i])) continue;
seen++;
if (seen === target) return i;
}
return -1;
}
export function estimateTokens(text) {
const value = String(text || "");
if (!value) return 0;
let cjk = 0;
let other = 0;
for (const ch of value) {
if (ch.codePointAt(0) >= 0x3000) cjk++;
else other++;
}
return Math.ceil(cjk * 1.5 + other / 4);
}
export function truncateToTokens(text, budget) {
const value = String(text || "");
const limit = Math.max(0, Math.floor(Number(budget) || 0));
if (!value || limit <= 0) return "";
if (estimateTokens(value) <= limit) return value;
let lo = 0;
let hi = value.length;
while (lo < hi) {
const mid = Math.ceil((lo + hi) / 2);
if (estimateTokens(value.slice(0, mid)) <= limit) lo = mid;
else hi = mid - 1;
}
return value.slice(0, lo);
}
function partText(part) {
if (!part || typeof part !== "object") return "";
if (typeof part.text === "string") return part.text;
if (part.type === "tool") {
const payload = {
name: part.tool_name,
input: part.tool_input,
output: part.tool_output,
status: part.tool_status,
};
try {
return JSON.stringify(payload);
} catch {
return String(part.tool_name || part.tool_output || "");
}
}
return flattenValue(part);
}
export function estimatePayloadTokens(payload) {
if (!payload || typeof payload !== "object") return 0;
if (typeof payload.content === "string") return estimateTokens(payload.content);
if (Array.isArray(payload.parts)) {
return estimateTokens(payload.parts.map(partText).filter(Boolean).join("\n\n"));
}
if (Array.isArray(payload.content)) {
return estimateTokens(payload.content.map(partText).filter(Boolean).join("\n\n"));
}
return 0;
}
export function buildOverviewMessage(
overview,
firstKeptTs = 0,
budget = DEFAULT_CONFIG.takeoverOverviewBudget,
recoveryHint = "",
) {
const raw = String(overview || "");
const truncated = truncateToTokens(raw, budget);
const body = truncated === raw ? raw : `${truncated}\n...(truncated)`;
const timestamp = Number.isFinite(Number(firstKeptTs)) ? Number(firstKeptTs) - 1 : 0;
return {
role: "user",
content:
`${OVERVIEW_MARKER} Earlier conversation was archived to OpenViking and summarized below. ` +
`Use OpenViking for related context when its tools are available.\n\n${body}${recoveryHint}`,
timestamp,
};
}
export function countUndeliveredForSession(pendingEntries, sid) {
if (!sid) return 0;
let count = 0;
for (const item of Array.isArray(pendingEntries) ? pendingEntries : []) {
const entry = asEntry(item);
if (entry?.type === "addMessage" && entry.sessionId === sid) count++;
}
return count;
}
/** Derive the parent history directory only for the server's archive shape. */
export function deriveHistoryUri(archiveUri) {
const cleaned = String(archiveUri || "").trim().replace(/\/+$/, "");
const m = /^(.*\/history)\/archive_\d+$/.exec(cleaned);
return m ? m[1] : "";
}
/**
* Read a commit response and decide whether it archived what this takeover
* meant to trim.
*
* The boundary may only advance behind a commit that actually wrote an archive:
* `status: "skipped"` (nothing to archive), an explicit `archived: false`, or a
* missing `archive_uri` all mean there is nothing to point the overview at, and
* the old `/context` summary must never be reused to fill the gap. Older server
* builds omit `status`/`archived` but still return `archive_uri`; those are
* accepted on the URI alone.
*/
export function commitOutcome(committed) {
if (!committed || typeof committed !== "object") {
return { accepted: false, reason: "no_result" };
}
const status = typeof committed.status === "string" ? committed.status : "";
if (status === "skipped") {
return { accepted: false, reason: `skipped:${committed.reason || "unknown"}` };
}
if (status && status !== "accepted") {
return { accepted: false, reason: `status:${status}` };
}
if (committed.archived === false) {
return { accepted: false, reason: "not_archived" };
}
const archiveUri =
typeof committed.archive_uri === "string" ? committed.archive_uri.trim() : "";
if (!archiveUri) {
return { accepted: false, reason: "no_archive_uri" };
}
return {
accepted: true,
reason: "accepted",
archiveUri,
taskId: typeof committed.task_id === "string" ? committed.task_id : "",
};
}
function archiveName(archiveUri) {
const parts = String(archiveUri || "").replace(/\/+$/, "").split("/");
return parts[parts.length - 1] || "the archive";
}
/** A user-facing reading of a `skipped` commit. */
export function describeSkip(reason, keepRecentCount = 0) {
if (reason === "all_within_keep_window") {
return (
`nothing new to archive: the server session holds at most the ${keepRecentCount} most recent ` +
"messages this commit keeps, and anything older was already archived (all_within_keep_window)"
);
}
if (reason === "no_messages") return "nothing to archive: the server session has no live messages (no_messages)";
return `the server skipped the archive (${reason || "unknown"})`;
}
export class TakeoverCore {
constructor({ config = {}, io = {} } = {}) {
this.config = takeoverConfig(config);
this.io = {
flush: io.flush || (async () => true),
// Sync the current branch before committing so the archive covers exactly
// the messages this trim is about to drop; not every caller has synced.
syncBranch: io.syncBranch || (async () => ({ added: 0, tokens: 0, allDelivered: true })),
commit: io.commit || (async () => null),
// Read the Working Memory of a SPECIFIC archive by its uri, so the summary
// provably belongs to this commit and not to an older /context archive.
readArchiveOverview: io.readArchiveOverview || (async () => null),
// An archive's terminal state from its server markers: "completed",
// "failed", "pending", or null when it cannot be asked.
archiveState: io.archiveState || (async () => null),
// How many messages the capture path would actually send for a slice of
// the branch — the server keep_recent_count is a message count, not a
// user-turn count, and it must exclude system/custom/filtered entries.
captureCount: io.captureCount || (() => 0),
persistEntry: io.persistEntry || (() => {}),
getWatermark: io.getWatermark || (() => 0),
// Messages OpenViking will never receive (4xx / retries exhausted): a
// capture gap that must stop takeover cutting across it.
droppedCount: io.droppedCount || (() => 0),
availableTools: io.availableTools || (() => []),
sleep: io.sleep || ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
now: io.now || (() => Date.now()),
// Why the last commit request returned no result (HTTP status and the
// server's message), so a manual commit can say more than "failed".
lastCommitError: io.lastCommitError || (() => ""),
log: io.log || (() => {}),
};
// The covered prefix ends at this pi entry, inclusive. Entry ids survive
// restarts, compaction and branch navigation and identify one entry in
// every view of the session, so the boundary needs no turn count or content
// fingerprint — the same reason the recall ledger keys on them.
this.coveredThroughEntryId = "";
// Display only: user turns the boundary covers in the active context.
this.coveredUserTurns = 0;
this.overview = "";
this.pendingTokens = 0;
this.lastSeenUserTurns = 0;
this.syncedEntryCount = 0;
this.committing = false;
this.lastPersisted = "";
this.archiveUri = "";
this.historyUri = "";
// A commit that archived but whose Working Memory was not ready in time:
// { archiveUri, taskId, historyUri, coveredThroughEntryId, coveredUserTurns,
// frozenTokens, nativeCompaction? }. While set, no second takeover commit
// runs — later turns only re-check this same archive.
this.pendingArchive = null;
// A permanent delivery gap in this session: the archive is missing messages,
// so the boundary must never advance past it. Survives restart.
this.captureGap = false;
// A count-based boundary (0.4.1 and earlier) — the first N user turns of the context, plus the
// fingerprint of the last covered message once it was learned — converted
// to an entry id the first time the context hook sees the session.
this.legacyBoundary = null;
// Whether the last context actually carried the boundary, so leaving and
// re-entering the covered branch is logged once, not on every request.
this.boundaryApplied = false;
// Why the last commitAndAdvance() returned false, for `/viking commit` to
// show; empty after a success. Not persisted.
this.lastFailure = "";
}
get enabled() {
return this.config.takeoverEnabled;
}
get state() {
return {
coveredThroughEntryId: this.coveredThroughEntryId,
coveredUserTurns: this.coveredUserTurns,
overview: this.overview,
pendingTokens: this.pendingTokens,
lastSeenUserTurns: this.lastSeenUserTurns,
syncedEntryCount: this.syncedEntryCount,
committing: this.committing,
pendingArchive: this.pendingArchive,
captureGap: this.captureGap,
archiveUri: this.archiveUri,
historyUri: this.historyUri,
};
}
restore(entries) {
for (let i = (Array.isArray(entries) ? entries.length : 0) - 1; i >= 0; i--) {
const entry = entries[i];
const isTakeoverEntry =
(entry?.type === "custom" && entry.customType === TAKEOVER_ENTRY_TYPE) ||
entry?.customType === TAKEOVER_ENTRY_TYPE ||
entry?.type === TAKEOVER_ENTRY_TYPE;
const data = isTakeoverEntry ? entry.data : null;
if (!data || typeof data !== "object") continue;
this.coveredThroughEntryId = typeof data.coveredThroughEntryId === "string" ? data.coveredThroughEntryId : "";
this.coveredUserTurns = Math.max(0, Math.floor(Number(data.coveredUserTurns) || 0));
this.legacyBoundary = !this.coveredThroughEntryId && this.coveredUserTurns > 0
? {
coveredUserTurns: this.coveredUserTurns,
fingerprint: typeof data.fingerprint === "string" ? data.fingerprint : null,
}
: null;
this.overview = typeof data.overview === "string" ? data.overview : "";
this.pendingTokens = Math.max(0, Math.floor(Number(data.pendingTokens) || 0));
this.lastSeenUserTurns = Math.max(0, Math.floor(Number(data.lastSeenUserTurns) || 0));
this.syncedEntryCount = Math.max(0, Math.floor(Number(data.syncedEntryCount) || 0));
// New fields default to safe absences so an old persisted entry restores.
this.pendingArchive = restorePendingArchive(data.pendingArchive);
this.captureGap = data.captureGap === true;
this.archiveUri = typeof data.archiveUri === "string" ? data.archiveUri : "";
this.historyUri = typeof data.historyUri === "string" ? data.historyUri : "";
this.lastPersisted = JSON.stringify(this.persistedState());
this.log(
`takeover: restored boundary ${this.coveredThroughEntryId || (this.legacyBoundary ? `at ${this.coveredUserTurns} user turns (count-based)` : "none")}` +
`, ${this.pendingTokens} pending tokens` +
(this.pendingArchive ? `, pending archive ${this.pendingArchive.archiveUri}` : "") +
(this.captureGap ? ", capture gap set" : ""),
);
return this.state;
}
return this.state;
}
/**
* Replace the covered prefix of the `context` hook's messages with the
* archive overview. `branch` is pi's `getBranch()`: the boundary is an entry
* id, found in pi's context projection of the branch and mapped onto the
* messages through the first user turn it keeps.
*/
transformContext(messages, branch = []) {
const list = Array.isArray(messages) ? messages : [];
this.lastSeenUserTurns = countUserTurns(list);
if (!this.enabled || !this.overview) return list;
const entries = projectContextEntries(currentBranch(branch));
if (this.legacyBoundary) this.adoptLegacyBoundary(list, entries);
if (!this.coveredThroughEntryId) return list;
const cut = this.locateCut(list, entries);
if (cut <= 0) {
if (this.boundaryApplied) {
this.log("takeover: covered prefix is not in the active context; sending it in full");
}
this.boundaryApplied = false;
return list;
}
this.boundaryApplied = true;
const kept = list.slice(cut);
const firstKeptTs = typeof kept[0]?.timestamp === "number" ? kept[0].timestamp : 1;
// Preserve every system message in the covered region, in its original
// order. On pi >= 0.86 the transcript carries the base prompt and its tool
// declarations as the leading `system` message, and mid-conversation tool
// additions/removals, section updates and appended instructions as later
// ones (see @earendil-works/pi-ai `getCurrentTools`/`getCurrentSystemMessage`).
// Slicing them off with the covered turns would strip the model's tools and
// instructions — the overview only stands in for the conversation, never for
// the system state. On 0.80.3 the transcript has no system messages here, so
// this preserves nothing and the behaviour is unchanged; on 0.87 the host
// reconciles tools against the executable set on every request, so keeping
// the existing declarations introduces no duplicate. System messages inside
// the retained tail are left where they are, not hoisted in front.
const coveredSystem = [];
for (let i = 0; i < cut; i++) {
if (list[i]?.role === "system") coveredSystem.push(list[i]);
}
return [
...coveredSystem,
buildOverviewMessage(
this.overview, firstKeptTs, this.config.takeoverOverviewBudget, this.recoveryHint(),
),
...kept,
];
}
/**
* Index in `messages` of the first message kept after the covered prefix,
* or -1 when the prefix is not in the active context (another branch, or pi
* compacted past it) or no user turn follows it yet. It never clears the
* boundary, so returning to the covered branch applies it again.
*/
locateCut(messages, entries) {
const through = entries.findIndex((entry) => entry?.id === this.coveredThroughEntryId);
if (through < 0) return -1;
let covered = 0;
for (let i = 0; i <= through; i++) {
if (isUserEntry(entries[i])) covered++;
}
// Messages can sit between the prefix and the next user turn: what a run
// added after a keepRecentTurns-0 commit, or a branch summary `/tree` left
// at the boundary. No archive covers them, so they are kept.
const between = [];
for (let i = through + 1; i < entries.length; i++) {
const entry = entries[i];
if (!isUserEntry(entry)) {
const role = contextRole(entry);
if (role) between.push(role);
continue;
}
let cut = findUserMessage(messages, entry.message, covered);
if (cut <= 0) return -1;
// Step back over them role by role; the hook shows no system messages on
// pi >= 0.87 and hoisted ones stay put anyway. A mismatch means the two
// views disagree, and nothing is trimmed.
for (let j = between.length - 1; j >= 0; j--) {
do cut--; while (cut >= 0 && messages[cut]?.role === "system");
if (cut < 0 || messages[cut]?.role !== between[j]) return -1;
}
if (cut > 0) this.coveredUserTurns = covered;
return cut;
}
return -1;
}
/**
* Convert a count-based boundary to an entry id: locate it the way 0.4.1 did — the
* (N+1)-th user turn of the context, checked against the learned fingerprint
* of the message before it — then take the entry in front of that turn. A
* boundary that no longer matches is dropped, as 0.4.1 itself would have.
*/
adoptLegacyBoundary(messages, entries) {
const { coveredUserTurns, fingerprint } = this.legacyBoundary;
this.legacyBoundary = null;
const cut = findBoundaryIndex(messages, coveredUserTurns);
if (cut <= 0 || (fingerprint !== null && fingerprintMessage(messages[cut - 1]) !== fingerprint)) {
this.resetBoundary("count-based boundary no longer matches the context");
return;
}
const users = entries.filter(isUserEntry);
const kept = findUserMessage(users.map((entry) => entry.message), messages[cut], coveredUserTurns);
const keptIdx = kept >= 0 ? entries.indexOf(users[kept]) : -1;
const throughId = keptIdx > 0 ? entries[keptIdx - 1]?.id : undefined;
if (typeof throughId !== "string" || !throughId) {
this.resetBoundary("count-based boundary has no entry id");
return;
}
this.coveredThroughEntryId = throughId;
this.coveredUserTurns = coveredUserTurns;
this.log(`takeover: count-based boundary adopted through entry ${throughId}`);
}
async onTurnSynced(estTokens, branch = [], { deadline } = {}) {
if (!this.enabled) return false;
// A commit already archived and is only waiting for its Working Memory:
// account for this turn, then check that one instead of opening another.
// Its resolution subtracts only the frozen token snapshot, leaving this new
// pressure for the next boundary.
this.pendingTokens += Math.max(0, Math.floor(Number(estTokens) || 0));
if (this.pendingArchive) return this.commitAndAdvance(branch, { deadline });
if (this.pendingTokens < this.config.takeoverTokenThreshold) return false;
// Whether the branch has enough user turns to leave a keep-recent tail is
// decided by freezeBoundary on pi's context projection of the branch.
return this.commitAndAdvance(branch, { deadline });
}
/**
* Check a pending archive once, outside `turn_end` — `before_agent_start`
* calls this so a summary that finished between prompts (or between `pi -p`
* / `pi -c` processes) trims the very next request.
*/
async resumePending(branch = [], { deadline } = {}) {
if (!this.enabled || this.committing || !this.pendingArchive) return false;
const until = this.deadlineFrom(deadline);
if (this.remaining(until) < 2 * READ_RESERVE_MS) return false;
this.committing = true;
try {
return await this.resolvePendingArchive(branch);
} catch (error) {
this.log(`takeover: pending archive check failed (${errorMessage(error)}); boundary held`);
return false;
} finally {
this.committing = false;
}
}
/**
* Advance the boundary behind a confirmed archive, or resume the archive a
* previous attempt left pending. Never advances across a capture gap and
* never reuses an old /context summary.
*
* Everything here runs inside a pi event handler, and hosts cap those at
* 30s: omp logs `handler timed out`, drops the result and lets the handler
* run on (#5275). So nothing waits for a summary — the archive is committed,
* its overview read once, and later turns check it again — and the drain and
* the commit only get what is left of `deadline`.
*/
async commitAndAdvance(branch = [], { deadline } = {}) {
if (!this.enabled) return this.fail("takeover is disabled");
if (this.committing) return this.fail("another commit is still running");
const until = this.deadlineFrom(deadline);
this.committing = true;
this.lastFailure = "";
try {
if (this.pendingArchive) return await this.resolvePendingArchive(branch);
return await this.beginArchive(branch, until);
} catch (error) {
this.log(`takeover: archive preparation failed (${errorMessage(error)}); boundary held`);
return this.fail(`archive preparation failed: ${errorMessage(error)}`);
} finally {
this.committing = false;
}
}
/** Freeze a boundary, confirm delivery, commit, and read this archive's summary once. */
async beginArchive(branch, until) {
if (this.captureGap) {
this.log("takeover: capture gap present; boundary held, native compaction stays with pi");
return this.fail("some messages never reached OpenViking (capture gap); takeover is off for this session");
}
// One snapshot is frozen, synced and counted, so the archive and
// keep_recent_count describe the same entries even if pi appends more
// while this commit is in flight.
const snapshot = currentBranch(branch);
const frozen = this.freezeBoundary(snapshot);
if (!frozen) {
this.log("takeover: no advanceable boundary; commit skipped");
return this.fail(
`nothing new to archive: the context needs more than ${this.config.takeoverKeepRecentTurns} ` +
"user turns beyond the last archive",
);
}
const delivered = await this.confirmDelivery(snapshot, until);
if (!delivered) {
return this.fail(
this.captureGap
? "some messages never reached OpenViking (capture gap); takeover is off for this session"
: "queued messages are not delivered yet; retry once the server is reachable",
);
}
const left = this.remaining(until);
if (left < COMMIT_RESERVE_MS) {
this.log("takeover: handler budget spent before commit; commit postponed");
return this.fail("ran out of time delivering messages; retry");
}
const committed = await this.io.commit({
queueOnFailure: false,
enableWorkingMemory: true,
keepRecentCount: frozen.keepRecentCount,
timeoutMs: left - READ_RESERVE_MS,
});
const outcome = commitOutcome(committed);
if (!outcome.accepted) {
// No archive was written: hold the boundary, keep the token pressure so
// the next threshold crossing retries, and never fall back to /context.
if (outcome.reason === "no_result") {
this.log("takeover: commit failed; boundary held");
const detail = String(this.io.lastCommitError() || "").trim();
return this.fail(`the server rejected the commit${detail ? ` (${detail})` : ""}`);
}
if (outcome.reason.startsWith("skipped:")) {
const reason = outcome.reason.slice(8);
this.log(`takeover: archive skipped (${reason}); boundary held`);
return this.fail(describeSkip(reason, frozen.keepRecentCount));
}
this.log(`takeover: commit returned no usable archive (${outcome.reason}); boundary held`);
return this.fail(`the server returned no archive (${outcome.reason})`);
}
if (committed.effective_enable_working_memory !== true) {
return this.fail("Working Memory was not enabled for this commit; using native context");
}
const archive = {
...frozen,
archiveUri: outcome.archiveUri,
taskId: outcome.taskId,
historyUri: deriveHistoryUri(outcome.archiveUri),
};
// Phase 2 writes the summary in the background and usually needs longer
// than a handler may wait, so this is one read, not a poll.
const overview = this.remaining(until) >= READ_RESERVE_MS
? await this.readOverviewOnce(archive.archiveUri)
: "";
if (overview) return this.advanceTo(branch, archive, overview);
// Accepted but not summarized yet: remember exactly this archive and its
// frozen boundary, keep local history, and let later turns and prompts
// re-check the SAME archive rather than committing again.
this.pendingArchive = {
archiveUri: archive.archiveUri,
taskId: archive.taskId || "",
historyUri: archive.historyUri,
coveredThroughEntryId: archive.coveredThroughEntryId,
coveredUserTurns: archive.coveredUserTurns,
frozenTokens: archive.frozenTokens,
};
this.persist();
this.log(`takeover: ${archive.archiveUri} committed, Working Memory not ready; checking on later turns`);
return false;
}
/**
* Sync the snapshot, then drain the queue: an empty queue alone does not
* prove delivery — the just-extracted turn may not have been sent yet. A
* permanent loss on either step records the capture gap.
*/
async confirmDelivery(snapshot, until) {
const synced = await this.io.syncBranch(snapshot);
if ((synced?.permanentFailures || 0) > 0 || this.io.droppedCount() > 0) {
this.markCaptureGap();
return false;
}
// A queued transient failure is allowed to proceed to the bounded drainer,
// which gets the handler time the commit does not need; only the barrier's
// final state proves that every current-session message reached the server.
const flushed = await this.io.flush(Math.max(0, this.remaining(until) - COMMIT_RESERVE_MS));
// The drain may have exhausted retries or failed to rewrite a queue entry.
// Record that permanent gap even when an orphaned processing file keeps the
// barrier closed.
if (this.io.droppedCount() > 0) {
this.markCaptureGap();
return false;
}
if (!flushed) {
this.log("takeover: flush barrier closed; commit postponed");
return false;
}
return true;
}
/**
* One check of the archive a prior attempt committed: advance once its
* summary is there, and drop it once the archive is terminal without one — a
* failed summary, or a server with Working Memory disabled, would otherwise
* hold takeover on this archive for the rest of the session.
*/
async resolvePendingArchive(branch) {
const pending = this.pendingArchive;
if (!pending) return false;
// A branch switch or a pi compaction during the wait can take the frozen
// prefix out of the active context; the advance is then abandoned, never
// moved somewhere else.
if (!pending.nativeCompaction && !this.inActiveContext(branch, pending.coveredThroughEntryId)) {
this.pendingArchive = null;
this.persist();
this.log("takeover: frozen boundary no longer in history; advance abandoned");
return this.fail("the archived turns left the active context (branch switch or compaction); advance abandoned");
}
let overview = await this.readOverviewOnce(pending.archiveUri);
if (!overview) {
const state = await this.io.archiveState(pending.archiveUri);
if (!TERMINAL_ARCHIVE_STATES.has(state)) return false;
// The summary may have landed between the two reads.
overview = await this.readOverviewOnce(pending.archiveUri);
if (!overview) {
this.pendingArchive = null;
// Wait for fresh pressure before the next archive, so a server that
// never writes Working Memory does not get a commit on every turn.
this.pendingTokens = Math.max(0, this.pendingTokens - (Number(pending.frozenTokens) || 0));
this.persist();
this.log(`takeover: ${pending.archiveUri} ${state} without Working Memory; boundary held`);
return this.fail(
`${archiveName(pending.archiveUri)} ${state} on the server without a Working Memory summary ` +
"(check the server log for the Phase 2 error)",
);
}
}
if (pending.nativeCompaction) {
// Pi already ran its own compaction after the earlier hook returned
// undefined. This archive must not install a second takeover boundary
// over Pi's new context, and the recovery hint keeps naming the archive
// the current overview came from.
this.pendingArchive = null;
this.pendingTokens = Math.max(0, this.pendingTokens - (Number(pending.frozenTokens) || 0));
this.persist();
this.log(`takeover: native-compaction archive completed at ${pending.archiveUri}`);
return this.fail(`${archiveName(pending.archiveUri)} completed after pi compacted natively; nothing to trim`);
}
return this.advanceTo(branch, pending, overview);
}
/** Move the boundary to a summarized archive, re-checking the frozen prefix first. */
advanceTo(branch, frozen, overview) {
if (!this.inActiveContext(branch, frozen.coveredThroughEntryId)) {
this.pendingArchive = null;
this.persist();
this.log("takeover: frozen boundary no longer in history; advance abandoned");
return this.fail("the archived turns left the active context (branch switch or compaction); advance abandoned");
}
this.coveredThroughEntryId = frozen.coveredThroughEntryId;
this.coveredUserTurns = frozen.coveredUserTurns;
this.legacyBoundary = null;
this.overview = overview;
this.archiveUri = frozen.archiveUri;
this.historyUri = frozen.historyUri || "";
this.pendingArchive = null;
// Subtract only the pressure this trim froze; tokens accrued while waiting
// for the summary belong to the next boundary and are not cleared.
this.pendingTokens = Math.max(0, this.pendingTokens - (Number(frozen.frozenTokens) || 0));
this.syncedEntryCount = Math.max(0, Math.floor(Number(this.io.getWatermark()) || 0));
this.persist();
this.log(
`takeover: boundary advanced to ${this.coveredUserTurns} user turns ` +
`(through ${this.coveredThroughEntryId}) via ${frozen.archiveUri}`,
);
return true;
}
/** Record why a commit did not advance, for `/viking commit` to show. */
fail(reason) {
this.lastFailure = String(reason || "");
return false;
}
inActiveContext(branch, entryId) {
if (!entryId) return false;
return projectContextEntries(currentBranch(branch)).some((entry) => entry?.id === entryId);
}
deadlineFrom(deadline) {
const value = Number(deadline);
return Number.isFinite(value) ? value : this.io.now() + HANDLER_BUDGET_MS;
}
remaining(until) {
return until - this.io.now();
}
/**
* Compute the candidate boundary on pi's context projection of the branch:
* keep the last `takeoverKeepRecentTurns` user turns, archive the rest.
* Returns the entry the covered prefix ends at, the exact keep_recent_count
* and the token snapshot, or null when there is nothing new to archive.
*/
freezeBoundary(branch) {
const raw = currentBranch(branch);
const entries = projectContextEntries(raw);
const users = [];
entries.forEach((entry, i) => {
if (isUserEntry(entry)) users.push(i);
});
const keep = this.config.takeoverKeepRecentTurns;
if (users.length <= keep) return null;
// The prefix ends right in front of the oldest kept user turn, or at the
// tip when nothing is kept; it then applies from the next user turn on.
const through = keep > 0 ? users[users.length - keep] - 1 : entries.length - 1;
const id = entries[through]?.id;
if (typeof id !== "string" || !id) return null;
// Only a prefix longer than the current one is worth an archive. A current
// boundary outside the active context (another branch, or compacted away)
// does not hold the new one back.
if (this.coveredThroughEntryId &&
entries.findIndex((entry) => entry?.id === this.coveredThroughEntryId) >= through) {
return null;
}
// keep_recent_count is a server message count: the capture payloads the
// branch holds after the covered prefix, in the order sync sent them —
// system, custom and filtered entries excluded.
const rawThrough = raw.findIndex((entry) => entry?.id === id);
return {
coveredThroughEntryId: id,
coveredUserTurns: users.length - keep,
keepRecentCount: Math.max(0, Math.floor(Number(this.io.captureCount(raw.slice(rawThrough + 1))) || 0)),
frozenTokens: this.pendingTokens,
};
}
markCaptureGap() {
if (this.captureGap) return;
this.captureGap = true;
this.pendingArchive = null;
this.persist();
this.log("takeover: capture gap recorded; takeover disabled for this session, native compaction stays with pi");
}
recordCaptureGap() {
this.markCaptureGap();
}
/**
* Answer pi's compaction with this session's own archive summary. Pi needs
* the summary now, so this is the one place that polls — but only until the
* handler deadline. Unless it returns a compaction, it changes no takeover
* state: pi may still cancel or fail its own compaction, and the boundary
* must then keep applying to the uncompacted context.
*/
async handleBeforeCompact(preparation = {}, branch = [], { deadline } = {}) {
if (!this.enabled || this.committing) return undefined;
if (!preparation.firstKeptEntryId) return undefined;
// A gap means the archive is missing messages, so an OpenViking summary
// cannot faithfully cover pi's cut — let pi compact natively.
if (this.captureGap) return undefined;
// A pending takeover archive covers only its frozen head; it cannot stand
// in for the broader range pi selected. Never create a second archive while
// the first one is unresolved.
if (this.pendingArchive) return undefined;
if (preparation.signal?.aborted) return undefined;
const until = this.deadlineFrom(deadline);
this.committing = true;
try {
// Sync the latest branch first; native compaction archives all captured
// history (keepRecentCount 0) and hands pi its own firstKeptEntryId, so
// the summary and the retained tail may overlap.
if (!(await this.confirmDelivery(currentBranch(branch), until))) return undefined;
if (preparation.signal?.aborted) return undefined;
const left = this.remaining(until);
if (left < COMMIT_RESERVE_MS) {
this.log("takeover: handler budget spent before native compaction commit; using pi compaction");
return undefined;
}
const committed = await this.io.commit({
queueOnFailure: false, enableWorkingMemory: true, keepRecentCount: 0, timeoutMs: left - READ_RESERVE_MS,
});
const outcome = commitOutcome(committed);
if (!outcome.accepted) {
if (outcome.reason === "no_result") {
this.log("takeover: native compaction commit failed; using pi compaction");
} else if (outcome.reason.startsWith("skipped:")) {
this.log(`takeover: native compaction archive skipped (${outcome.reason.slice(8)}); using pi compaction`);
} else {
this.log(`takeover: native compaction returned no usable archive (${outcome.reason}); using pi compaction`);
}
return undefined;
}
// Older servers may ignore the request field. Never trim native history
// unless the server confirmed the policy used by this exact commit.
if (committed.effective_enable_working_memory !== true) {
this.log("takeover: Working Memory was not enabled; using pi compaction");
return undefined;
}
const historyUri = deriveHistoryUri(outcome.archiveUri);
const { overview, terminal } = await this.pollArchiveOverview(outcome.archiveUri, preparation.signal, until);
if (!overview && terminal) return undefined;
if (!overview || this.remaining(until) <= 0) {
// The server accepted the archive, but this compaction attempt cannot
// wait any longer. Remember it so ordinary takeover will not create a
// second archive, and hand this compaction back to pi.
this.pendingArchive = {
archiveUri: outcome.archiveUri,
taskId: outcome.taskId || "",
historyUri,
coveredThroughEntryId: "",
coveredUserTurns: 0,
frozenTokens: this.pendingTokens,
nativeCompaction: true,
};
this.persist();
this.log(`takeover: ${outcome.archiveUri} not summarized in time; using pi compaction`);
return undefined;
}
this.overview = overview;
this.archiveUri = outcome.archiveUri;
this.historyUri = historyUri;
this.resetBoundary("pi compaction absorbed boundary");
this.pendingTokens = 0;
this.syncedEntryCount = Math.max(0, Math.floor(Number(this.io.getWatermark()) || 0));
this.persist();
return {
compaction: {
summary:
`${OVERVIEW_MARKER}\n${this.truncatedOverview()}` +
this.recoveryHint(outcome.archiveUri, historyUri),
firstKeptEntryId: preparation.firstKeptEntryId,
tokensBefore: Number(preparation.tokensBefore) || 0,
details: { source: "openviking" },
},
};
} catch (error) {
this.log(`takeover: native compaction archive failed (${errorMessage(error)}); using pi compaction`);
return undefined;
} finally {
this.committing = false;
}
}
async shutdown() {
if (!this.enabled) return;
this.syncedEntryCount = Math.max(0, Math.floor(Number(this.io.getWatermark()) || 0));
this.persist();
}
resetBoundary(reason = "reset") {
if (this.coveredThroughEntryId || this.legacyBoundary) {
this.log(`takeover: boundary reset (${reason})`);
}
this.coveredThroughEntryId = "";
this.coveredUserTurns = 0;
this.legacyBoundary = null;
this.boundaryApplied = false;
}
truncatedOverview() {
const raw = String(this.overview || "");
const truncated = truncateToTokens(raw, this.config.takeoverOverviewBudget);
return truncated === raw ? raw : `${truncated}\n...(truncated)`;
}
/** Append a non-budgeted source recovery hint only when both required tools exist. */
recoveryHint(archiveUri = this.archiveUri, historyUri = this.historyUri) {
const tools = new Set(this.io.availableTools());
if (!tools.has("openviking_list") || !tools.has("openviking_read")) return "";
const archive = String(archiveUri || "").trim();
const history = String(historyUri || "").trim();
if (!archive || !history) return "";
return (
`\n\nArchived capture: ${archive}\n` +
`This contains captured historical messages, not the unfiltered Pi transcript. ` +
`Semantic search does not retrieve archive source verbatim. Use openviking_list on ${history} ` +
`to locate archives, then openviking_read with uris=["${archive}/messages.jsonl"], ` +
`offset and limit to read it in chunks.` +
(tools.has("openviking_grep") ? ` Use openviking_grep only as an optional locator.` : "")
);
}
persistedState() {
const rawWatermark = Number(this.io.getWatermark());
const watermark = Number.isFinite(rawWatermark)
? Math.max(0, Math.floor(rawWatermark))
: this.syncedEntryCount;
return {
coveredThroughEntryId: this.coveredThroughEntryId,
// Also what a 0.4.1 reader restores its boundary from after a downgrade.
coveredUserTurns: this.coveredUserTurns,
// A count-based boundary not yet adopted keeps its fingerprint across restarts.
...(this.legacyBoundary ? { fingerprint: this.legacyBoundary.fingerprint } : {}),
overview: truncateToTokens(this.overview, this.config.takeoverOverviewBudget),
pendingTokens: this.pendingTokens,
lastSeenUserTurns: this.lastSeenUserTurns,
syncedEntryCount: watermark,
pendingArchive: this.pendingArchive,
captureGap: this.captureGap,
archiveUri: this.archiveUri,
historyUri: this.historyUri,
};
}
persist() {
try {
const state = this.persistedState();
const key = JSON.stringify(state);
if (key === this.lastPersisted) return;
this.io.persistEntry(TAKEOVER_ENTRY_TYPE, state);
this.lastPersisted = key;
} catch {
// Best effort. A missed state entry only means the next process sees full history.
}
}
/** One read of an archive's `.overview.md`; "" until its non-empty body lands. */
async readOverviewOnce(archiveUri, timeoutMs = READ_RESERVE_MS) {
const uri = String(archiveUri || "").trim();
if (!uri) return "";
try {
const value = await this.io.readArchiveOverview(uri, timeoutMs);
return typeof value === "string" ? value.trim() : "";
} catch (error) {
this.log(`takeover: archive overview read failed for ${uri} (${errorMessage(error)})`);
return "";
}
}
/**
* Poll one archive's `.overview.md` — `takeoverOverviewPollMax` reads,
* `takeoverOverviewPollMs` apart — and stop early at `until`.
*/
async pollArchiveOverview(archiveUri, signal, until = Infinity) {
for (let i = 0; i < this.config.takeoverOverviewPollMax; i++) {
if (signal?.aborted || this.remaining(until) <= 0) break;
const overview = await this.readOverviewOnce(archiveUri, Math.min(READ_RESERVE_MS, this.remaining(until)));
if (signal?.aborted) break;
if (overview) return { overview, terminal: false };
if (this.remaining(until) <= 0) break;
const state = await this.io.archiveState(archiveUri, Math.min(2 * READ_RESERVE_MS, this.remaining(until)));
if (signal?.aborted) break;
if (TERMINAL_ARCHIVE_STATES.has(state)) {
// The summary may land between the first read and the terminal marker.
// A final read closes that race; a terminal archive must not stay pending.
if (this.remaining(until) <= 0) return { overview: "", terminal: true };
const finalOverview = await this.readOverviewOnce(archiveUri, Math.min(READ_RESERVE_MS, this.remaining(until)));
return { overview: signal?.aborted ? "" : finalOverview, terminal: true };
}
if (i === this.config.takeoverOverviewPollMax - 1 || this.config.takeoverOverviewPollMs <= 0) break;
if (this.remaining(until) < this.config.takeoverOverviewPollMs + READ_RESERVE_MS) break;
await this.io.sleep(this.config.takeoverOverviewPollMs);
}
return { overview: "", terminal: false };
}
log(message) {
try {
this.io.log(message);
} catch {
// Logging must not affect pi's context path.
}
}
}
/** Rehydrate a persisted pending-archive record, dropping malformed shapes. */
function restorePendingArchive(value) {
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
const archiveUri = typeof value.archiveUri === "string" ? value.archiveUri.trim() : "";
if (!archiveUri) return null;
return {
archiveUri,
taskId: typeof value.taskId === "string" ? value.taskId : "",
historyUri: typeof value.historyUri === "string" ? value.historyUri : "",
coveredThroughEntryId: typeof value.coveredThroughEntryId === "string" ? value.coveredThroughEntryId : "",
coveredUserTurns: Math.max(0, Math.floor(Number(value.coveredUserTurns) || 0)),
frozenTokens: Math.max(0, Math.floor(Number(value.frozenTokens) || 0)),
...(value.nativeCompaction === true ? { nativeCompaction: true } : {}),
};
}
function errorMessage(error) {
return error instanceof Error ? error.message : String(error || "unknown");
}