#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
import { TriggerAction, type IIIClient } from "iii-sdk";
import type { RawObservation, CompressedObservation, HookPayload, Origin, Session } from "../types.js";
const TOOL_HOOKS = new Set(["pre_tool_use", "post_tool_use", "post_tool_failure"]);
import { KV, generateId } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { emitLiveEvent } from "../viewer/live-events.js";
import { addSessionToProjectIndex } from "../state/session-index.js";
import { indexObservationSession } from "../state/obs-index.js";
import { scrubRecord } from "./privacy.js";
import { DedupMap, recordDedupSkip } from "./dedup.js";
import { afterKeyedLock, withKeyedLock } from "../state/keyed-mutex.js";
import { getFinalizeIdleMs, isAutoCompressEnabled } from "../config.js";
import { buildSyntheticCompression } from "./compress-synthetic.js";
import { isCaptureKey } from "../capture/event-record.js";
import { claimBackfillPrompt, isBackfillPrompt, recordLivePrompt } from "../capture/prompt-ledger.js";
import { getSearchIndex, getVectorIndex, scheduleIndexSave, vectorIndexAddGuarded } from "./search.js";
import { getAgentId } from "../config.js";
import { logger } from "../logger.js";
import { saveImageToDisk } from "../utils/image-store.js";
import { withoutObservationSource } from "./observation-source.js";
import { budgetLiveObservationSource } from "./observation-source-budget.js";
import { markCompressionDispatchFailed, pendingCompression } from "./compression-state.js";
import { requestSummaryRefresh } from "./summary-state.js";
import { scheduleSessionFinalize } from "./session-finalize.js";
export function extractImage(d: unknown): string | undefined {
if (!d) return undefined;
if (typeof d === "string") {
if (d.startsWith("data:image/") || d.startsWith("iVBORw0KGgo") || d.startsWith("/9j/")) {
return d;
}
return undefined;
}
if (typeof d === "object" && d !== null) {
const obj = d as Record<string, unknown>;
if (typeof obj["image_data"] === "string") return obj["image_data"];
if (typeof obj["image_path"] === "string") return obj["image_path"];
if (typeof obj["imageBase64"] === "string") return obj["imageBase64"];
if (typeof obj["imagePath"] === "string") return obj["imagePath"];
for (const key of Object.keys(obj)) {
const match = extractImage(obj[key]);
if (match) return match;
}
}
return undefined;
}
export async function restoreIndexEntries(obs: CompressedObservation): Promise<void> {
if (typeof obs.title !== "string" || !obs.sessionId) return;
const search = getSearchIndex();
if (!search.has(obs.id)) {
search.add(obs);
scheduleIndexSave();
}
const vectors = getVectorIndex();
if (vectors && !vectors.has(obs.id)) {
await vectorIndexAddGuarded(obs.id, obs.sessionId, obs.title + " " + (obs.narrative || ""), {
kind: "synthetic",
logId: obs.id,
});
}
}
export function registerObserveFunction(
sdk: IIIClient,
kv: StateKV,
dedupMap?: DedupMap,
maxObservationsPerSession?: number,
): void {
sdk.registerFunction("mem::observe",
async (payload: HookPayload) => {
if (
!payload?.sessionId ||
typeof payload.sessionId !== "string" ||
!payload.hookType ||
typeof payload.hookType !== "string" ||
!payload.timestamp ||
typeof payload.timestamp !== "string"
) {
return {
success: false,
error:
"Invalid payload: sessionId, hookType, and timestamp are required",
};
}
const durable =
typeof payload.observationId === "string" && /^obs_[A-Za-z0-9_]{8,80}$/.test(payload.observationId);
const obsId = durable ? payload.observationId! : generateId("obs");
let dedupHash: string | undefined;
if (dedupMap && typeof payload.eventId !== "string") {
const dataIsObject =
typeof payload.data === "object" && payload.data !== null;
const d = dataIsObject
? (payload.data as Record<string, unknown>)
: {};
const toolName = (d["tool_name"] as string) || payload.hookType;
// Hash the full payload when tool_input is absent so distinct
// events never collapse onto one key.
const dedupInput =
d["tool_input"] !== undefined
? d["tool_input"]
: dataIsObject
? d
: payload.data;
const dedupOutput =
d["tool_input"] !== undefined
? d["tool_response"] ?? d["tool_output"] ?? d["output"] ?? d["error"]
: undefined;
dedupHash = dedupMap.computeHash(
payload.sessionId,
toolName,
dedupInput,
dedupOutput,
);
if (dedupMap.isDuplicate(dedupHash)) {
recordDedupSkip();
return { deduplicated: true, sessionId: payload.sessionId };
}
}
const serializedData = JSON.stringify(payload.data);
const sanitizedRaw = serializedData === undefined
? undefined
: scrubRecord(JSON.parse(serializedData), { scrubKeys: true });
let originChannel: Origin["channel"] = "agent";
if (payload.hookType === "prompt_submit") originChannel = "user";
else if (TOOL_HOOKS.has(payload.hookType)) originChannel = "tool";
const raw: RawObservation = {
id: obsId,
sessionId: payload.sessionId,
timestamp: payload.timestamp,
hookType: payload.hookType,
raw: sanitizedRaw,
origin: {
channel: originChannel,
capturedAt: payload.timestamp,
},
...(typeof payload.eventId === "string" ? { eventId: payload.eventId } : {}),
...(isCaptureKey(payload.captureKey) ? { captureKey: payload.captureKey } : {}),
};
let extractedImage: string | undefined;
if (typeof sanitizedRaw === "object" && sanitizedRaw !== null) {
const d = sanitizedRaw as Record<string, unknown>;
if (TOOL_HOOKS.has(payload.hookType)) {
raw.toolName = d["tool_name"] as string | undefined;
raw.toolInput = d["tool_input"];
raw.toolOutput = d["tool_output"] ?? d["error"];
if (raw.origin && raw.toolName) raw.origin.detail = raw.toolName;
}
if (payload.hookType === "prompt_submit") {
raw.userPrompt = d["prompt"] as string | undefined;
}
if (payload.hookType === "stop") {
const response = d["assistant_response"] ?? d["last_assistant_message"] ?? d["response"];
if (typeof response === "string") raw.assistantResponse = response;
const originalBytes = d["last_assistant_message_bytes"];
if (typeof originalBytes === "number" && Number.isSafeInteger(originalBytes) && originalBytes >= 0) {
raw.assistantResponseBytes = originalBytes;
}
}
extractedImage = extractImage(sanitizedRaw);
if (extractedImage) {
raw.modality = (raw.toolInput || raw.toolOutput || raw.userPrompt) ? "mixed" : "image";
}
} else if (typeof sanitizedRaw === "string") {
extractedImage = extractImage(sanitizedRaw);
if (extractedImage) {
raw.modality = "image";
}
}
const pendingImageData = extractedImage;
const submittedPrompt =
payload.hookType === "prompt_submit" && typeof raw.userPrompt === "string" ? raw.userPrompt : undefined;
const backfillPrompt = submittedPrompt && isBackfillPrompt(payload.data) ? submittedPrompt : undefined;
const livePrompt = submittedPrompt && !backfillPrompt ? submittedPrompt : undefined;
const promptRef = durable || typeof payload.eventId !== "string" ? obsId : payload.eventId;
return withKeyedLock(`obs:${payload.sessionId}`, async () => {
const existing = await kv.list<CompressedObservation>(KV.observations(payload.sessionId));
const stored = durable ? existing.find((o) => o?.id === obsId) : undefined;
if (stored) {
if (livePrompt) await recordLivePrompt(kv, payload.sessionId, livePrompt, promptRef);
await restoreIndexEntries(stored);
return { observationId: obsId, deduplicated: true, existing: true };
}
if (backfillPrompt && (await claimBackfillPrompt(kv, payload.sessionId, backfillPrompt, promptRef))) {
recordDedupSkip();
return { deduplicated: true, sessionId: payload.sessionId };
}
if (maxObservationsPerSession && maxObservationsPerSession > 0) {
if (existing.length >= maxObservationsPerSession) {
return {
success: false,
error: `Session observation limit reached (${maxObservationsPerSession})`,
};
}
}
// Existing session is the source of truth for agentId (even
// undefined). Env AGENT_ID only fires when no session row
// exists yet — otherwise an unscoped session would get
// retroactively scoped by a later AGENT_ID export.
const existingSession = await kv.get<{
agentId?: string;
observationCount?: number;
firstPrompt?: string;
status?: Session["status"];
}>(KV.sessions, payload.sessionId);
const inheritedAgentId = existingSession
? existingSession.agentId
: getAgentId();
if (inheritedAgentId) {
raw.agentId = inheritedAgentId;
}
if (pendingImageData && (pendingImageData.startsWith("data:image/") || pendingImageData.startsWith("iVBORw0KGgo") || pendingImageData.startsWith("/9j/"))) {
const { filePath, bytesWritten } = await saveImageToDisk(pendingImageData);
raw.imageData = filePath;
const { incrementImageRef } = await import("./image-refs.js");
await incrementImageRef(kv, filePath);
sdk.trigger({
function_id: "mem::disk-size-delta",
payload: { deltaBytes: bytesWritten },
action: TriggerAction.Void(),
});
if (process.env["AGENTMEMORY_IMAGE_EMBEDDINGS"] === "true") {
sdk.trigger({
function_id: "mem::vision-embed",
payload: {
imageRef: filePath,
sessionId: payload.sessionId,
observationId: obsId,
},
action: TriggerAction.Void(),
});
}
}
const autoCompress = isAutoCompressEnabled();
const finalizeIdleMs = getFinalizeIdleMs();
if (existingSession && existingSession.status !== "active") {
const dueAt = Date.now() + finalizeIdleMs;
await requestSummaryRefresh(kv, payload.sessionId, dueAt);
if (finalizeIdleMs > 0) await scheduleSessionFinalize(kv, payload.sessionId, dueAt);
}
const synthetic = budgetLiveObservationSource(buildSyntheticCompression(raw), existing);
if (autoCompress) synthetic.compression = pendingCompression();
try {
await kv.set(KV.observations(payload.sessionId), obsId, synthetic);
await indexObservationSession(kv, obsId, payload.sessionId).catch(
(err) => {
logger.warn("observation index update failed", {
obsId,
sessionId: payload.sessionId,
error: err instanceof Error ? err.message : String(err),
});
},
);
} catch (error) {
if (raw.imageData) {
// Roll back the ref taken above. decrementImageRef deletes the file
// only when no other observation still references it (deduped images
// survive) and emits the disk-size delta itself — deleting the file
// directly here would orphan shared images and leave a stale ref.
// If the rollback itself fails, log it but still surface the
// original write error (the more useful failure to diagnose).
try {
const { decrementImageRef } = await import("./image-refs.js");
await decrementImageRef(kv, sdk, raw.imageData);
} catch (rollbackError) {
logger.error("Failed to roll back image ref after observation write failure", {
imageRef: raw.imageData,
error: rollbackError instanceof Error ? rollbackError.message : String(rollbackError),
});
}
}
throw error;
}
if (dedupMap && dedupHash) {
dedupMap.record(dedupHash);
}
if (livePrompt) await recordLivePrompt(kv, payload.sessionId, livePrompt, promptRef);
emitLiveEvent({
id: `raw-${obsId}`,
type: "raw_observation",
data: { type: "raw", observation: raw, sessionId: payload.sessionId },
});
const session = existingSession;
if (session) {
const updates: Array<{ type: "set" | "remove"; path: string; value?: unknown }> = [
{ type: "set", path: "updatedAt", value: new Date().toISOString() },
{
type: "set",
path: "observationCount",
value: (session.observationCount || 0) + 1,
},
];
if (session.status !== "active" && finalizeIdleMs > 0) {
updates.push({ type: "set", path: "status", value: "active" });
updates.push({ type: "remove", path: "endedAt" });
}
if (!session.firstPrompt && typeof raw.userPrompt === "string") {
const trimmed = raw.userPrompt.replace(/\s+/g, " ").trim();
if (trimmed.length > 0) {
updates.push({
type: "set",
path: "firstPrompt",
value: trimmed.slice(0, 200),
});
}
}
await kv.update(KV.sessions, payload.sessionId, updates);
} else if (
typeof payload.project === "string" &&
payload.project.trim().length > 0 &&
typeof payload.cwd === "string" &&
payload.cwd.trim().length > 0
) {
// OpenCode (and any plugin that skips POST /session/start)
// can fire observations before the session record exists. Without
// an implicit create, those observations stack up but
// `memory_sessions` never lists them, and summarize bails with
// "Session not found for summarize". Create the session now from
// the observation payload — but only when project + cwd are
// present (HookPayload contract). Older test payloads without
// those fields keep their original no-op behaviour.
const trimmedPrompt =
typeof raw.userPrompt === "string"
? raw.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200)
: undefined;
const ts = new Date().toISOString();
const startedAt = payload.timestamp ?? ts;
await kv.set(KV.sessions, payload.sessionId, {
id: payload.sessionId,
project: payload.project,
cwd: payload.cwd,
startedAt,
updatedAt: ts,
status: "active",
observationCount: 1,
...(inheritedAgentId ? { agentId: inheritedAgentId } : {}),
...(trimmedPrompt && trimmedPrompt.length > 0
? { firstPrompt: trimmedPrompt }
: {}),
});
await addSessionToProjectIndex(kv, payload.project, {
id: payload.sessionId,
startedAt,
...(inheritedAgentId ? { agentId: inheritedAgentId } : {}),
}).catch((err) => {
logger.warn("session index update failed", {
sessionId: payload.sessionId,
error: err instanceof Error ? err.message : String(err),
});
});
}
getSearchIndex().add(synthetic);
scheduleIndexSave();
await vectorIndexAddGuarded(
synthetic.id,
synthetic.sessionId,
synthetic.title + " " + (synthetic.narrative || ""),
{ kind: "synthetic", logId: synthetic.id },
);
emitLiveEvent({
id: `compressed-${obsId}`,
type: "compressed_observation",
data: {
type: "compressed",
observation: withoutObservationSource(synthetic),
sessionId: payload.sessionId,
},
});
if (autoCompress) {
afterKeyedLock(`obs:${payload.sessionId}`, async () => {
try {
await sdk.trigger({
function_id: "mem::compress",
payload: { observationId: obsId, sessionId: payload.sessionId },
action: TriggerAction.Void(),
});
} catch {
await markCompressionDispatchFailed(kv, synthetic);
}
});
}
logger.info("Observation captured", {
obsId,
sessionId: payload.sessionId,
hook: payload.hookType,
compress: autoCompress ? "llm" : "synthetic",
});
return { observationId: obsId };
});
},
);
}