Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
/**
* Pi OpenViking Extension
*
* Integrates pi with an OpenViking context database for persistent,
* cross-session memory. Syncs conversation turns to OV, recalls
* relevant memories on each prompt, and commits sessions for long-term
* memory extraction.
*
* Design informed by: OpenClaw (synchronous recall), Claude Code plugin
* (most mature, production-hardened), Hermes (anti-pattern: stale prefetch).
*/
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { isCaptureEnabled } from "./shared/capture-utils.mjs";
import { createLogger } from "./shared/debug-log.mjs";
import { EXTENSION_VERSION, loadConfig, buildBridgeProxyConfig, type OVConfig } from "./config.js";
import { OVClient } from "./client.js";
import { RecallManager } from "./recall.js";
import { RecallLedger } from "./lib/recall-ledger.mjs";
import { SyncManager } from "./sync.js";
import { buildProfileBlock } from "./shared/profile-inject.mjs";
import { isBypassed } from "./shared/session-model.mjs";
import { guardVikingUriToolCall, noticeVikingUriToolResult } from "./lib/uri-guard-adapter.mjs";
import { createMcpBridge, DEFAULT_HANDSHAKE_BUDGET_MS } from "./lib/mcp-bridge.mjs";
import { registerMcpTools } from "./tools.js";
import { createTakeoverManager } from "./takeover.js";
import { HANDLER_BUDGET_MS, describeSkip } from "./lib/takeover-core.mjs";
/** This extension's directory, published for the experimental fork's probe. */
const EXTENSION_DIR = dirname(fileURLToPath(import.meta.url));
export default async function (pi: ExtensionAPI) {
// --- Load config ---
const cwd = process.cwd();
const config = loadConfig(cwd);
if (!config.enabled) return;
// Shared key (`shared/config-schema.mjs`), already honoured by opencode: the
// MCP tool surface is off, while recall, sync and takeover carry on.
const mcpEnabled = (config as any).mcpEnabled !== false;
// Installed as an extension rather than a package, so pi does not scan
// skills/ on its own. The skills teach the MCP tools, so they follow them.
if (mcpEnabled) {
pi.on("resources_discover", async () => ({ skillPaths: [join(EXTENSION_DIR, "skills")] }));
}
// Env overrides
// --- Initialize modules ---
const client = new OVClient(config);
const sync = new SyncManager(client, config);
const recall = new RecallManager(
client,
config,
() => sync.sessionId,
// The ledger keeps request prefixes byte-stable for provider prompt
// caches (#4137); it is per pi session and opened once the id is known.
config.recallLedger ? new RecallLedger() : null,
);
const logger = createLogger("pi", {
debug: Boolean(config.debugLogPath),
debugLogPath: config.debugLogPath,
});
const takeover = createTakeoverManager({
pi, client, sync, config,
log: (message: string) => logger.log("takeover", message),
});
// Re-read credentials before connecting or invoking a tool.
const bridge = mcpEnabled
? createMcpBridge({
readConfig: () => buildBridgeProxyConfig(loadConfig(cwd)),
// No "memory" in the name: every plugin identity is moving to plain
// `openviking`, and a new identifier starts out on the new spelling.
clientInfo: { name: "openviking-pi", version: EXTENSION_VERSION },
})
: null;
// Session state
let connected = false;
let bypassed = false;
let profileBlock = "";
let archiveOverview = "";
let toolsReady = false;
let toolsRegistered = false;
let closed = false;
const marker = { dir: EXTENSION_DIR };
const toolNames: string[] = [];
let toolFailureNotified = false;
let compacted = false;
let started = false;
let startPromise: Promise<void> | null = null;
/**
* Take a finished handshake: register what the server listed, or report the
* failure once.
*
* The tool list is never refreshed later in the session. pi has no
* `unregisterTool`, and adding or removing a tool mid-session rewrites the
* system prompt's tool section, which invalidates the provider's cached
* prefix. A server that gained or lost a tool takes effect in the next pi
* session.
*/
const adoptBridgeTools = (
ctx: any,
state: { connected: boolean; error: string | null },
): void => {
if (!bridge || closed || toolsRegistered) return;
if (state.connected) {
toolNames.push(...registerMcpTools(pi, bridge));
toolsRegistered = true;
toolsReady = toolNames.length > 0;
logger.log("mcp_tools", { registered: toolNames.length });
return;
}
// A dead, unauthorized or MCP-less server costs the session its tools and
// nothing else, so it is said once and not repeated every turn.
if (toolFailureNotified) return;
toolFailureNotified = true;
if (config.logLevel === "silent") return;
ctx?.ui?.notify?.(
`OpenViking: no tools this session — ${state.error || "handshake failed"}${rootKeyHint(state.error)}`,
"warning",
);
};
/** The `/viking` line about the tool surface: a count, or the last failure. */
const toolsStatusLine = (): string => {
if (!bridge) return "tools: disabled by mcpEnabled";
if (toolNames.length) return `tools: ${toolNames.length}`;
const error = bridge.state.error;
if (!error) return "tools: none (handshake not finished)";
return `tools: none — ${error}${rootKeyHint(error)}`;
};
// ================================================================
// Event Handlers
// ================================================================
const start = async (ctx: any): Promise<void> => {
if (started || closed) return;
if (startPromise) return startPromise;
startPromise = (async () => {
// Bypass check
if (isBypassed(config, { cwd: process.cwd() })) {
bypassed = true;
started = true;
return;
}
// Health check
connected = await client.health();
if (closed) return;
if (!connected) {
if (config.logLevel === "info") {
ctx.ui.notify("OpenViking: server not reachable", "warning");
}
return;
}
// What the experimental fork probes to stand down. Set as soon as the
// session is live, independently of any tool: a /mcp 401/403 leaves this
// extension with zero tools while it still writes the OpenViking session,
// and that is exactly the case the fork must not join.
(globalThis as any).__OPENVIKING_PI_EXTENSION__ = marker;
// Discover tools alongside REST startup; failure leaves REST available.
const handshake = bridge ? bridge.connect(DEFAULT_HANDSHAKE_BUDGET_MS) : null;
// Ensure OV session
const piSessionId = ctx.sessionManager.getSessionId();
recall.openLedger(piSessionId);
const ok = await sync.ensureSession(piSessionId);
if (!ok) {
if (config.logLevel !== "silent") {
ctx.ui.notify("OpenViking: failed to create session", "error");
}
return;
}
// Restore takeover state — sync watermark, pending archive and capture
// gap — before replaying any queued messages, so recovery and replay
// cannot disagree about what has been delivered (plan §4).
const branch = typeof ctx.sessionManager.getBranch === "function"
? ctx.sessionManager.getBranch()
: [];
if (config.takeoverEnabled) {
takeover.restore(branch);
sync.restoreWatermark(takeover.state.syncedEntryCount, branch);
}
await sync.replayPending();
if (config.takeoverEnabled && sync.droppedCount > 0) {
takeover.recordCaptureGap();
}
// Profile injection
profileBlock = await buildSessionProfileBlock(client, config);
if (!config.takeoverEnabled && config.resumeArchiveInject && sync.sessionId) {
// Resume rehydration — fetch archive overview if session was previously committed.
archiveOverview = await fetchArchiveOverview(client, sync.sessionId, config);
}
// Join the handshake and publish whatever the server listed (also needed
// for pi -c continuations, which never fire session_start).
if (handshake) adoptBridgeTools(ctx, await handshake);
if (closed) return;
updateStatus(ctx, connected, 0, sync.sessionId, config, takeover.state, toolsReady);
started = true;
if (config.logLevel === "info") {
ctx.ui.notify(`OpenViking connected (${piSessionId.slice(0, 8)}...)`, "info");
}
})().finally(() => {
startPromise = null;
});
return startPromise;
};
// --- session_start ---
pi.on("session_start", async (event, ctx) => {
// Fire-and-forget: the OV chain (health check, session ensure, profile
// build) costs ~2s against the remote server; blocking session_start on it
// delays every pi startup. start() is memoized via startPromise, so
// before_agent_start awaits the same in-flight chain before the first
// provider request — the first turn still gets profile + recall.
void start(ctx).catch((error) => {
logger.logError("session_start", error);
});
});
// --- before_agent_start ---
pi.on("before_agent_start", async (event, ctx) => {
const deadline = Date.now() + HANDLER_BUDGET_MS;
// Read before awaiting, because this is what separates the retry below
// from the attempt start() makes: on the turn that runs the startup chain
// this is false, and from the next turn on it is true. Testing `started`
// after the await would be true on turn one as well, and the retry would
// fire a second handshake in the same turn — a real one, since the bridge
// clears its in-flight promise as soon as a failed attempt settles. That
// costs the first turn a second handshake budget before recall is even
// queued, for an answer start() already has.
const wasStarted = started;
// session_start doesn't fire for pi -c continuations.
await start(ctx);
// Retry the handshake when the session is live but toolless — a server
// started after pi, or a credential fixed mid-session. `bridge` is null
// exactly when `mcpEnabled` is false, and a bypassed directory or a failed
// health check never gets here: bypass means this directory does not touch
// OpenViking at all. `connect()` re-handshakes only after a failure; after
// a success it hands back the state it already has.
if (bridge && wasStarted && connected && !bypassed && !closed && !toolsRegistered) {
adoptBridgeTools(ctx, await bridge.connect(DEFAULT_HANDSHAKE_BUDGET_MS));
updateStatus(ctx, connected, 0, sync.sessionId, config, takeover.state, toolsReady);
}
if (!connected || bypassed || closed) return;
// A summary that finished since the last turn — or in the last `pi -p`
// process — trims this prompt's request already.
if (config.takeoverEnabled) {
await takeover.resumePending(() => ctx.sessionManager.getBranch(), { deadline });
}
// Queue recall for the context hook. Pi renders the user message before
// that hook, so recall latency does not delay the message appearing.
recall.queueSearch(event.prompt);
// Compose system prompt additions
const parts: string[] = [];
if (profileBlock) parts.push(profileBlock);
if (!config.takeoverEnabled && config.resumeArchiveInject && archiveOverview && (compacted || archiveOverview.trim())) {
parts.push(archiveOverview);
}
// Generated from what actually registered, so it can never name a tool the
// server does not have. Omitted entirely when the handshake produced none,
// which keeps the prefix free of a promise nothing can keep.
if (toolNames.length) {
parts.push(`OpenViking tools (use these for viking:// URIs): ${toolNames.join(", ")}.`);
}
const additions = parts.join("\n\n");
if (!additions) return;
return {
systemPrompt: event.systemPrompt + "\n\n" + additions,
};
});
// --- context ---
pi.on("context", async (event, ctx) => {
if (!connected || bypassed) return;
// Keep recall synchronous with the provider request so the current prompt
// still receives current-query memory, without blocking user-message UI.
await recall.searchPending();
// The entry IDs are an optional optimization for replaying the recall
// ledger. Compatible hosts may omit buildContextEntries(), so fail closed
// to nullable IDs rather than guessing from another SessionManager API.
const sessionManager = ctx.sessionManager;
const entries = typeof sessionManager?.buildContextEntries === "function"
? sessionManager.buildContextEntries()
: [];
const userEntryIds = entries
.filter((entry: unknown): entry is { id?: unknown; type: "message"; message: { role: "user" } } => {
if (!entry || typeof entry !== "object") return false;
if (!("type" in entry) || !("message" in entry)) return false;
const type = entry.type;
const message = entry.message;
return type === "message" &&
!!message &&
typeof message === "object" &&
"role" in message &&
message.role === "user";
})
.map((entry): string | undefined =>
typeof entry.id === "string" ? entry.id : undefined
);
const messageIds = new WeakMap<object, string>();
let userIndex = 0;
for (const message of event.messages as any[]) {
if (message?.role !== "user") continue;
const entryId = userEntryIds[userIndex++];
if (entryId && typeof message === "object") {
messageIds.set(message, entryId);
}
}
// Takeover's boundary is a branch entry id; it maps it onto these messages
// through pi's context projection of the branch.
const afterTakeover = config.takeoverEnabled
? takeover.transformContext(
event.messages as any,
typeof sessionManager?.getBranch === "function" ? sessionManager.getBranch() : [],
)
: event.messages;
const messages = recall.injectRecall(
afterTakeover,
(message) => messageIds.get(message) ?? null,
);
return { messages };
});
// --- tool_call ---
pi.on("tool_call", async (event, _ctx) => {
const decision = guardVikingUriToolCall(event);
if (!decision) return;
return decision;
});
// --- tool_result ---
pi.on("tool_result", async (event, _ctx) => {
const notice = noticeVikingUriToolResult(event);
if (!notice) return;
return notice;
});
// --- turn_end ---
pi.on("turn_end", async (event, ctx) => {
if (!connected || bypassed || !isCaptureEnabled(config)) return;
// The host's 30s cap covers this whole handler, sync included.
const deadline = Date.now() + HANDLER_BUDGET_MS;
const branch = ctx.sessionManager.getBranch();
const result = await sync.syncBranch(branch);
logger.log("turn_end", { added: result.added, tokens: result.tokens });
if (result.permanentFailures > 0) takeover.recordCaptureGap();
// The branch is what the boundary is frozen against and re-confirmed on, so
// takeover needs it, not just the token delta.
await takeover.onTurnSynced(result.tokens, () => ctx.sessionManager.getBranch(), { deadline });
updateStatus(ctx, connected, result.added, sync.sessionId, config, takeover.state, toolsReady);
});
// --- session_before_compact ---
pi.on("session_before_compact", async (event, ctx) => {
if (!connected || bypassed || !isCaptureEnabled(config)) return;
if (config.takeoverEnabled) {
const deadline = Date.now() + HANDLER_BUDGET_MS;
const prep = (event as any)?.preparation ?? {};
// Native compaction syncs the latest branch, archives all captured
// history and reuses pi's own firstKeptEntryId, so hand it the branch.
return await takeover.handleBeforeCompact({
firstKeptEntryId: prep.firstKeptEntryId,
tokensBefore: prep.tokensBefore ?? 0,
signal: (event as any)?.signal,
}, () => ctx.sessionManager.getBranch(), { deadline });
}
const archiveId = await sync.commit();
compacted = true;
// Cache archive overview for rehydration after compaction
if (config.resumeArchiveInject && archiveId && sync.sessionId) {
archiveOverview = await fetchArchiveOverview(
client, sync.sessionId, config,
);
}
// Return nothing → pi proceeds with default compaction
});
// --- session_shutdown ---
pi.on("session_shutdown", async (_event, ctx) => {
closed = true;
if ((globalThis as any).__OPENVIKING_PI_EXTENSION__ === marker) {
delete (globalThis as any).__OPENVIKING_PI_EXTENSION__;
}
// Release MCP requests without preventing the final REST commit.
if (bridge) {
try {
await bridge.close();
} catch (error) {
logger.logError("session_shutdown", error);
}
}
if (!connected || bypassed) return;
await sync.shutdown();
if (config.takeoverEnabled) {
await takeover.shutdown();
} else if (isCaptureEnabled(config)) {
await sync.commit();
}
});
// --- agent_end ---
pi.on("agent_end", async (_event, _ctx) => {
recall.invalidate();
});
// ================================================================
// Commands
// ================================================================
pi.registerCommand("viking", {
description: "OpenViking status and manual operations. Use 'commit' to force a sync.",
handler: async (args, ctx) => {
if (!connected) {
ctx.ui.notify("OpenViking: not connected", "warning");
return;
}
if (args?.trim() === "commit") {
await sync.shutdown();
const commitResult = config.takeoverEnabled ? null : await sync.commit();
// Manual commit freezes and confirms the boundary against the current
// branch, exactly like the automatic path.
// A `skipped` result archived nothing; it is not a successful commit.
const skipped = commitResult?.status === "skipped";
const ok = config.takeoverEnabled
? await takeover.commitAndAdvance(() => ctx.sessionManager.getBranch())
: commitResult !== null && !skipped;
if (!ok && config.takeoverEnabled && takeover.state.pendingArchive) {
ctx.ui.notify(
"OpenViking: committed; the context is trimmed once the archive summary is ready",
"info",
);
} else if (ok) {
ctx.ui.notify(
"OpenViking: committed successfully" +
(commitResult?.trace_id ? ` (trace_id=${commitResult.trace_id})` : ""),
"info",
);
} else {
// Say why: a bare "commit failed" left users with nothing to act on.
const reason = config.takeoverEnabled
? takeover.lastFailure
: skipped
? describeSkip(String(commitResult?.reason || ""))
: sync.lastCommitError;
logger.log("commit", { ok: false, manual: true, reason: reason || "unknown" });
if (reason.startsWith("nothing")) {
// Not an error: the server already holds this history.
ctx.ui.notify(`OpenViking: ${reason}`, "warning");
} else {
ctx.ui.notify(`OpenViking: commit failed${reason ? ` — ${reason}` : ""}`, "error");
}
}
return;
}
// Status
const sid = sync.sessionId ?? "none";
const t = takeover.state;
const takeoverInfo = config.takeoverEnabled
? ` | takeover: ${t.coveredUserTurns}/${t.lastSeenUserTurns} turns archived, ~${t.pendingTokens} tokens pending`
: "";
ctx.ui.notify(
`OpenViking: ${connected ? "connected" : "disconnected"} | session: ${sid.slice(0, 12)}...`
+ `${takeoverInfo} | ${toolsStatusLine()}`,
"info",
);
},
});
}
// ================================================================
// Helper Functions
// ================================================================
/** Build the <openviking-context> profile block. */
async function buildSessionProfileBlock(
client: OVClient, config: OVConfig,
): Promise<string> {
try {
const profile = await buildProfileBlock(
(path, init, options) => client.fetchJSON(path, init, options),
config.profileTokenBudget,
config.peerId,
config,
);
if (!profile?.block) return "";
return [
'<openviking-context source="session-start">',
profile.block,
"</openviking-context>",
].join("\n");
} catch {
return "";
}
}
/** Fetch archive overview for rehydration using the session context API. */
async function fetchArchiveOverview(
client: OVClient, sessionId: string, config: OVConfig,
): Promise<string> {
try {
const ctx = await client.getSessionContext(sessionId, config.resumeContextBudget);
if (!ctx || !ctx.latest_archive_overview) return "";
return [
'<openviking-context source="session-archive">',
"<session-archive>",
ctx.latest_archive_overview,
"</session-archive>",
"</openviking-context>",
].join("\n");
} catch {
return "";
}
}
/**
* The extra line for a failed handshake that a root key explains.
*
* A root key authenticates fine and is then refused on `/mcp`, so the HTTP
* status is the only clue, and without this the operator sees a 403 with no
* way to act on it.
*/
function rootKeyHint(error: string | null): string {
return error && /\b403\b/.test(error)
? "\nA root key cannot reach /mcp — use a user or admin key."
: "";
}
function updateStatus(
ctx: any,
connected: boolean,
added: number,
sessionId: string | null,
config: OVConfig,
takeoverState?: { pendingTokens?: number; coveredUserTurns?: number },
toolsReady?: boolean,
): void {
const setter = ctx?.ui?.setStatus;
if (typeof setter !== "function") return;
const threshold = config.takeoverEnabled
? config.takeoverTokenThreshold
: config.commitTokenThreshold;
const pending = config.takeoverEnabled && takeoverState
? ` · ctx ${takeoverState.coveredUserTurns ?? 0} · ~${takeoverState.pendingTokens ?? 0}/${threshold}`
: ` · ✎ ${threshold}`;
// Only worth a segment when tools were expected and are missing: `OV ✓` with
// no tools is a working session whose OpenViking tools failed, and
// `mcpEnabled: false` asked for that and is not a fault.
const tools = connected && (config as any).mcpEnabled !== false && !toolsReady
? " · tools ✗"
: "";
const status = `${connected ? "OV ✓" : "OV ✗"}${tools} · ↩${added}${pending} · ${sessionId ? sessionId.slice(0, 12) : "none"}`;
try {
setter("openviking", status);
} catch {
// Best effort; pi API shape may vary across fast-moving versions.
}
}