Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

feat: v0.3.0 — self-evaluating agentic memory system

Rohit Ghumare57d6f087ade2
diff --git a/package.json b/package.json
index 1077696d..7d6adbb6 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "agentmemory",
- "version": "0.2.0",
+ "version": "0.3.0",
"description": "Persistent memory for AI coding agents, powered by iii-engine's three primitives",
"type": "module",
"main": "dist/index.js",
diff --git a/src/eval/metrics-store.ts b/src/eval/metrics-store.ts
new file mode 100644
index 00000000..5831261a
--- /dev/null
+++ b/src/eval/metrics-store.ts
@@ -0,0 +1,53 @@
+import type { FunctionMetrics } from "../types.js";
+import type { StateKV } from "../state/kv.js";
+import { KV } from "../state/schema.js";
+
+export class MetricsStore {
+ private cache = new Map<string, FunctionMetrics>();
+
+ constructor(private kv: StateKV) {}
+
+ async record(
+ functionId: string,
+ latencyMs: number,
+ success: boolean,
+ qualityScore?: number,
+ ): Promise<void> {
+ let m = this.cache.get(functionId);
+ if (!m) {
+ m = await this.kv.get<FunctionMetrics>(KV.metrics, functionId) ?? {
+ functionId,
+ totalCalls: 0,
+ successCount: 0,
+ failureCount: 0,
+ avgLatencyMs: 0,
+ avgQualityScore: 0,
+ };
+ }
+
+ const prev = m.totalCalls;
+ m.totalCalls += 1;
+ m.avgLatencyMs = (m.avgLatencyMs * prev + latencyMs) / m.totalCalls;
+ if (success) {
+ m.successCount += 1;
+ } else {
+ m.failureCount += 1;
+ }
+ if (qualityScore !== undefined) {
+ m.avgQualityScore = (m.avgQualityScore * prev + qualityScore) / m.totalCalls;
+ }
+
+ this.cache.set(functionId, m);
+ await this.kv.set(KV.metrics, functionId, m).catch(() => {});
+ }
+
+ async get(functionId: string): Promise<FunctionMetrics | null> {
+ return this.cache.get(functionId) ??
+ await this.kv.get<FunctionMetrics>(KV.metrics, functionId);
+ }
+
+ async getAll(): Promise<FunctionMetrics[]> {
+ if (this.cache.size > 0) return Array.from(this.cache.values());
+ return this.kv.list<FunctionMetrics>(KV.metrics).catch(() => []);
+ }
+}
diff --git a/src/eval/quality.ts b/src/eval/quality.ts
new file mode 100644
index 00000000..9f565d5a
--- /dev/null
+++ b/src/eval/quality.ts
@@ -0,0 +1,51 @@
+export function scoreCompression(obs: {
+ type?: string;
+ title?: string;
+ facts?: string[];
+ narrative?: string;
+ concepts?: string[];
+ importance?: number;
+}): number {
+ let score = 0;
+ if (obs.facts && obs.facts.length > 0) score += 25;
+ if (obs.facts && obs.facts.length >= 3) score += 10;
+ if (obs.narrative && obs.narrative.length >= 20) score += 20;
+ if (obs.narrative && obs.narrative.length >= 50) score += 5;
+ if (obs.title && obs.title.length >= 5 && obs.title.length <= 120) score += 15;
+ if (obs.concepts && obs.concepts.length > 0) score += 15;
+ if (obs.importance && obs.importance >= 1 && obs.importance <= 10) score += 10;
+ return Math.min(100, score);
+}
+
+export function scoreSummary(summary: {
+ title?: string;
+ narrative?: string;
+ keyDecisions?: string[];
+ filesModified?: string[];
+ concepts?: string[];
+}): number {
+ let score = 0;
+ if (summary.title && summary.title.length >= 5) score += 20;
+ if (summary.narrative && summary.narrative.length >= 20) score += 25;
+ if (summary.narrative && summary.narrative.length >= 100) score += 5;
+ if (summary.keyDecisions && summary.keyDecisions.length > 0) score += 20;
+ if (summary.filesModified && summary.filesModified.length > 0) score += 15;
+ if (summary.concepts && summary.concepts.length > 0) score += 15;
+ return Math.min(100, score);
+}
+
+export function scoreContextRelevance(
+ context: string,
+ project: string,
+): number {
+ let score = 0;
+ if (context.length > 0) score += 20;
+ if (project && context.toLowerCase().includes(project.toLowerCase())) score += 20;
+ if (context.includes("<")) score += 15;
+ const sectionCount = (context.match(/<\w+>/g) || []).length;
+ if (sectionCount >= 2) score += 15;
+ if (sectionCount >= 4) score += 10;
+ if (context.length >= 100) score += 10;
+ if (context.length >= 500) score += 10;
+ return Math.min(100, score);
+}
diff --git a/src/eval/schemas.ts b/src/eval/schemas.ts
new file mode 100644
index 00000000..6ce820e0
--- /dev/null
+++ b/src/eval/schemas.ts
@@ -0,0 +1,59 @@
+import { z } from "zod";
+
+const HookTypeEnum = z.enum([
+ "session_start", "prompt_submit", "pre_tool_use", "post_tool_use",
+ "post_tool_failure", "pre_compact", "subagent_start", "subagent_stop",
+ "notification", "task_completed", "stop", "session_end",
+]);
+
+const ObservationTypeEnum = z.enum([
+ "file_read", "file_write", "file_edit", "command_run", "search",
+ "web_fetch", "conversation", "error", "decision", "discovery",
+ "subagent", "notification", "task", "other",
+]);
+
+export const ObserveInputSchema = z.object({
+ hookType: HookTypeEnum,
+ sessionId: z.string().min(1),
+ project: z.string().min(1),
+ cwd: z.string().min(1),
+ timestamp: z.string().min(1),
+ data: z.unknown(),
+});
+
+export const CompressOutputSchema = z.object({
+ type: ObservationTypeEnum,
+ title: z.string().min(1).max(120),
+ subtitle: z.string().optional(),
+ facts: z.array(z.string()).min(1),
+ narrative: z.string().min(10),
+ concepts: z.array(z.string()),
+ files: z.array(z.string()),
+ importance: z.number().int().min(1).max(10),
+});
+
+export const SummaryOutputSchema = z.object({
+ title: z.string().min(1),
+ narrative: z.string().min(20),
+ keyDecisions: z.array(z.string()),
+ filesModified: z.array(z.string()),
+ concepts: z.array(z.string()),
+});
+
+export const SearchInputSchema = z.object({
+ query: z.string().min(1),
+ limit: z.number().int().positive().optional(),
+});
+
+export const ContextInputSchema = z.object({
+ sessionId: z.string().min(1),
+ project: z.string().min(1),
+ budget: z.number().positive().optional(),
+});
+
+export const RememberInputSchema = z.object({
+ content: z.string().min(1),
+ type: z.enum(["pattern", "preference", "architecture", "bug", "workflow", "fact"]).optional(),
+ concepts: z.array(z.string()).optional(),
+ files: z.array(z.string()).optional(),
+});
diff --git a/src/eval/self-correct.ts b/src/eval/self-correct.ts
new file mode 100644
index 00000000..68fc34db
--- /dev/null
+++ b/src/eval/self-correct.ts
@@ -0,0 +1,28 @@
+import type { MemoryProvider } from "../types.js";
+
+const STRICTER_SUFFIX = `
+
+IMPORTANT: Your previous response was invalid. Please ensure your output strictly follows the required XML format. Every required field must be present with valid values.`;
+
+export async function compressWithRetry(
+ provider: MemoryProvider,
+ systemPrompt: string,
+ userPrompt: string,
+ validator: (response: string) => { valid: boolean; errors?: string[] },
+ maxRetries = 1,
+): Promise<{ response: string; retried: boolean }> {
+ const first = await provider.compress(systemPrompt, userPrompt);
+ const result = validator(first);
+ if (result.valid) return { response: first, retried: false };
+
+ for (let i = 0; i < maxRetries; i++) {
+ const retry = await provider.compress(
+ systemPrompt + STRICTER_SUFFIX,
+ userPrompt,
+ );
+ const retryResult = validator(retry);
+ if (retryResult.valid) return { response: retry, retried: true };
+ }
+
+ return { response: first, retried: true };
+}
diff --git a/src/eval/validator.ts b/src/eval/validator.ts
new file mode 100644
index 00000000..21332ce6
--- /dev/null
+++ b/src/eval/validator.ts
@@ -0,0 +1,31 @@
+import type { z } from "zod";
+import type { EvalResult } from "../types.js";
+
+export function validateInput<T>(
+ schema: z.ZodType<T>,
+ data: unknown,
+ functionId: string,
+): { valid: true; data: T } | { valid: false; result: EvalResult } {
+ const parsed = schema.safeParse(data);
+ if (parsed.success) {
+ return { valid: true, data: parsed.data };
+ }
+ return {
+ valid: false,
+ result: {
+ valid: false,
+ errors: parsed.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`),
+ qualityScore: 0,
+ latencyMs: 0,
+ functionId,
+ },
+ };
+}
+
+export function validateOutput<T>(
+ schema: z.ZodType<T>,
+ data: unknown,
+ functionId: string,
+): { valid: true; data: T } | { valid: false; result: EvalResult } {
+ return validateInput(schema, data, functionId);
+}
diff --git a/src/functions/compress.ts b/src/functions/compress.ts
index 0ed1f0e7..ead2775c 100644
--- a/src/functions/compress.ts
+++ b/src/functions/compress.ts
@@ -14,6 +14,11 @@ import {
} from "../prompts/compression.js";
import { getXmlTag, getXmlChildren } from "../prompts/xml.js";
import { getSearchIndex } from "./search.js";
+import { CompressOutputSchema } from "../eval/schemas.js";
+import { validateOutput } from "../eval/validator.js";
+import { scoreCompression } from "../eval/quality.js";
+import { compressWithRetry } from "../eval/self-correct.js";
+import type { MetricsStore } from "../eval/metrics-store.js";
const VALID_TYPES = new Set<string>([
"file_read",
@@ -59,6 +64,7 @@ export function registerCompressFunction(
sdk: ISdk,
kv: StateKV,
provider: MemoryProvider,
+ metricsStore?: MetricsStore,
): void {
sdk.registerFunction(
{
@@ -71,6 +77,7 @@ export function registerCompressFunction(
raw: RawObservation;
}) => {
const ctx = getContext();
+ const startMs = Date.now();
const prompt = buildCompressionPrompt({
hookType: data.raw.hookType,
toolName: data.raw.toolName,
@@ -81,15 +88,42 @@ export function registerCompressFunction(
});
try {
- const response = await provider.compress(COMPRESSION_SYSTEM, prompt);
+ const validator = (response: string) => {
+ const parsed = parseCompressionXml(response);
+ if (!parsed) return { valid: false, errors: ["xml_parse_failed"] };
+ const result = validateOutput(
+ CompressOutputSchema,
+ parsed,
+ "mem::compress",
+ );
+ return result.valid
+ ? { valid: true }
+ : { valid: false, errors: result.result.errors };
+ };
+
+ const { response, retried } = await compressWithRetry(
+ provider,
+ COMPRESSION_SYSTEM,
+ prompt,
+ validator,
+ 1,
+ );
+
const parsed = parseCompressionXml(response);
if (!parsed) {
+ const latencyMs = Date.now() - startMs;
+ if (metricsStore) {
+ await metricsStore.record("mem::compress", latencyMs, false);
+ }
ctx.logger.warn("Failed to parse compression XML", {
obsId: data.observationId,
+ retried,
});
return { success: false, error: "parse_failed" };
}
+ const qualityScore = scoreCompression(parsed);
+
const compressed: CompressedObservation = {
id: data.observationId,
sessionId: data.sessionId,
@@ -112,15 +146,31 @@ export function registerCompressFunction(
data: { type: "compressed", observation: compressed },
});
+ const latencyMs = Date.now() - startMs;
+ if (metricsStore) {
+ await metricsStore.record(
+ "mem::compress",
+ latencyMs,
+ true,
+ qualityScore,
+ );
+ }
+
ctx.logger.info("Observation compressed", {
obsId: data.observationId,
type: compressed.type,
importance: compressed.importance,
+ qualityScore,
+ retried,
});
- return { success: true, compressed };
+ return { success: true, compressed, qualityScore };
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
+ const latencyMs = Date.now() - startMs;
+ if (metricsStore) {
+ await metricsStore.record("mem::compress", latencyMs, false);
+ }
ctx.logger.error("Compression failed", {
obsId: data.observationId,
error: msg,
diff --git a/src/functions/dedup.ts b/src/functions/dedup.ts
new file mode 100644
index 00000000..63f594d2
--- /dev/null
+++ b/src/functions/dedup.ts
@@ -0,0 +1,55 @@
+import { createHash } from "node:crypto";
+
+const TTL_MS = 5 * 60 * 1000;
+const CLEANUP_INTERVAL_MS = 60_000;
+
+interface DedupEntry {
+ hash: string;
+ expiresAt: number;
+}
+
+export class DedupMap {
+ private entries = new Map<string, DedupEntry>();
+ private cleanupTimer: ReturnType<typeof setInterval>;
+
+ constructor() {
+ this.cleanupTimer = setInterval(() => this.cleanup(), CLEANUP_INTERVAL_MS);
+ }
+
+ computeHash(sessionId: string, toolName: string, toolInput: unknown): string {
+ const input = typeof toolInput === "string"
+ ? toolInput.slice(0, 500)
+ : JSON.stringify(toolInput ?? "").slice(0, 500);
+ const raw = `${sessionId}:${toolName}:${input}`;
+ return createHash("sha256").update(raw).digest("hex");
+ }
+
+ isDuplicate(hash: string): boolean {
+ const entry = this.entries.get(hash);
+ if (!entry) return false;
+ if (Date.now() > entry.expiresAt) {
+ this.entries.delete(hash);
+ return false;
+ }
+ return true;
+ }
+
+ record(hash: string): void {
+ this.entries.set(hash, { hash, expiresAt: Date.now() + TTL_MS });
+ }
+
+ private cleanup(): void {
+ const now = Date.now();
+ for (const [key, entry] of this.entries) {
+ if (now > entry.expiresAt) this.entries.delete(key);
+ }
+ }
+
+ stop(): void {
+ clearInterval(this.cleanupTimer);
+ }
+
+ get size(): number {
+ return this.entries.size;
+ }
+}
diff --git a/src/functions/evict.ts b/src/functions/evict.ts
new file mode 100644
index 00000000..cfe7846b
--- /dev/null
+++ b/src/functions/evict.ts
@@ -0,0 +1,97 @@
+import type { ISdk } from "iii-sdk";
+import { getContext } from "iii-sdk";
+import type { Session, CompressedObservation, SessionSummary } from "../types.js";
+import { KV } from "../state/schema.js";
+import { StateKV } from "../state/kv.js";
+
+interface EvictionConfig {
+ staleSessionDays: number;
+ lowImportanceMaxDays: number;
+ lowImportanceThreshold: number;
+ maxObservationsPerProject: number;
+}
+
+const DEFAULTS: EvictionConfig = {
+ staleSessionDays: 30,
+ lowImportanceMaxDays: 90,
+ lowImportanceThreshold: 3,
+ maxObservationsPerProject: 10_000,
+};
+
+interface EvictionStats {
+ staleSessions: number;
+ lowImportanceObs: number;
+ capEvictions: number;
+ dryRun: boolean;
+}
+
+export function registerEvictFunction(sdk: ISdk, kv: StateKV): void {
+ sdk.registerFunction(
+ { id: "mem::evict", description: "Evict stale memories based on age and importance" },
+ async (data: { dryRun?: boolean }): Promise<EvictionStats> => {
+ const ctx = getContext();
+ const dryRun = data?.dryRun ?? false;
+
+ const configOverride = await kv.get<Partial<EvictionConfig>>(KV.config, "eviction").catch(() => null);
+ const cfg = { ...DEFAULTS, ...configOverride };
+
+ const now = Date.now();
+ const stats: EvictionStats = { staleSessions: 0, lowImportanceObs: 0, capEvictions: 0, dryRun };
+
+ const sessions = await kv.list<Session>(KV.sessions).catch(() => []);
+ const summaries = await kv.list<SessionSummary>(KV.summaries).catch(() => []);
+ const summaryIds = new Set(summaries.map((s) => s.sessionId));
+
+ for (const session of sessions) {
+ if (!session.startedAt) continue;
+ const age = now - new Date(session.startedAt).getTime();
+ const staleDays = cfg.staleSessionDays * 24 * 60 * 60 * 1000;
+ if (age > staleDays && !summaryIds.has(session.id)) {
+ stats.staleSessions++;
+ if (!dryRun) {
+ await kv.delete(KV.sessions, session.id).catch(() => {});
+ }
+ }
+ }
+
+ const projectObs = new Map<string, CompressedObservation[]>();
+ for (const session of sessions) {
+ const obs = await kv.list<CompressedObservation>(KV.observations(session.id)).catch(() => []);
+ const compressed = obs.filter((o) => o.title);
+
+ for (const o of compressed) {
+ if (!o.timestamp) continue;
+ const age = now - new Date(o.timestamp).getTime();
+ const maxAge = cfg.lowImportanceMaxDays * 24 * 60 * 60 * 1000;
+ if (age > maxAge && (o.importance ?? 5) < cfg.lowImportanceThreshold) {
+ stats.lowImportanceObs++;
+ if (!dryRun) {
+ await kv.delete(KV.observations(session.id), o.id).catch(() => {});
+ }
+ }
+ }
+
+ const project = session.project || "unknown";
+ const existing = projectObs.get(project) || [];
+ existing.push(...compressed);
+ projectObs.set(project, existing);
+ }
+
+ for (const [, obs] of projectObs) {
+ if (obs.length > cfg.maxObservationsPerProject) {
+ const sorted = obs.sort((a, b) => (a.importance ?? 5) - (b.importance ?? 5));
+ const toEvict = sorted.slice(0, obs.length - cfg.maxObservationsPerProject);
+ stats.capEvictions += toEvict.length;
+ if (!dryRun) {
+ for (const o of toEvict) {
+ await kv.delete(KV.observations(o.sessionId), o.id).catch(() => {});
+ }
+ }
+ }
+ }
+
+ ctx.logger.info("Eviction complete", { stats });
+ return stats;
+ },
+ );
+}
diff --git a/src/functions/observe.ts b/src/functions/observe.ts
index edb9481e..389c0bf9 100644
--- a/src/functions/observe.ts
+++ b/src/functions/observe.ts
@@ -4,8 +4,13 @@ import type { RawObservation, HookPayload, Session } from "../types.js";
import { KV, STREAM, generateId } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
import { stripPrivateData } from "./privacy.js";
+import { DedupMap } from "./dedup.js";
-export function registerObserveFunction(sdk: ISdk, kv: StateKV): void {
+export function registerObserveFunction(
+ sdk: ISdk,
+ kv: StateKV,
+ dedupMap?: DedupMap,
+): void {
sdk.registerFunction(
{
id: "mem::observe",
@@ -14,6 +19,24 @@ export function registerObserveFunction(sdk: ISdk, kv: StateKV): void {
async (payload: HookPayload) => {
const ctx = getContext();
const obsId = generateId("obs");
+
+ if (dedupMap) {
+ const d =
+ typeof payload.data === "object" && payload.data !== null
+ ? (payload.data as Record<string, unknown>)
+ : {};
+ const toolName = (d["tool_name"] as string) || payload.hookType;
+ const hash = dedupMap.computeHash(
+ payload.sessionId,
+ toolName,
+ d["tool_input"],
+ );
+ if (dedupMap.isDuplicate(hash)) {
+ return { deduplicated: true, sessionId: payload.sessionId };
+ }
+ dedupMap.record(hash);
+ }
+
let sanitizedRaw: unknown = payload.data;
try {
const jsonStr = JSON.stringify(payload.data);
diff --git a/src/functions/summarize.ts b/src/functions/summarize.ts
index 1933540c..17e2c96c 100644
--- a/src/functions/summarize.ts
+++ b/src/functions/summarize.ts
@@ -1,72 +1,143 @@
-import type { ISdk } from 'iii-sdk'
-import { getContext } from 'iii-sdk'
-import type { CompressedObservation, SessionSummary, MemoryProvider, Session } from '../types.js'
-import { KV } from '../state/schema.js'
-import { StateKV } from '../state/kv.js'
-import { SUMMARY_SYSTEM, buildSummaryPrompt } from '../prompts/summary.js'
-import { getXmlTag, getXmlChildren } from '../prompts/xml.js'
+import type { ISdk } from "iii-sdk";
+import { getContext } from "iii-sdk";
+import type {
+ CompressedObservation,
+ SessionSummary,
+ MemoryProvider,
+ Session,
+} from "../types.js";
+import { KV } from "../state/schema.js";
+import { StateKV } from "../state/kv.js";
+import { SUMMARY_SYSTEM, buildSummaryPrompt } from "../prompts/summary.js";
+import { getXmlTag, getXmlChildren } from "../prompts/xml.js";
+import { SummaryOutputSchema } from "../eval/schemas.js";
+import { validateOutput } from "../eval/validator.js";
+import { scoreSummary } from "../eval/quality.js";
+import type { MetricsStore } from "../eval/metrics-store.js";
-function parseSummaryXml(xml: string, sessionId: string, project: string, obsCount: number): SessionSummary | null {
- const title = getXmlTag(xml, 'title')
- if (!title) return null
+function parseSummaryXml(
+ xml: string,
+ sessionId: string,
+ project: string,
+ obsCount: number,
+): SessionSummary | null {
+ const title = getXmlTag(xml, "title");
+ if (!title) return null;
return {
sessionId,
project,
createdAt: new Date().toISOString(),
title,
- narrative: getXmlTag(xml, 'narrative'),
- keyDecisions: getXmlChildren(xml, 'decisions', 'decision'),
- filesModified: getXmlChildren(xml, 'files', 'file'),
- concepts: getXmlChildren(xml, 'concepts', 'concept'),
+ narrative: getXmlTag(xml, "narrative"),
+ keyDecisions: getXmlChildren(xml, "decisions", "decision"),
+ filesModified: getXmlChildren(xml, "files", "file"),
+ concepts: getXmlChildren(xml, "concepts", "concept"),
observationCount: obsCount,
- }
+ };
}
-export function registerSummarizeFunction(sdk: ISdk, kv: StateKV, provider: MemoryProvider): void {
+export function registerSummarizeFunction(
+ sdk: ISdk,
+ kv: StateKV,
+ provider: MemoryProvider,
+ metricsStore?: MetricsStore,
+): void {
sdk.registerFunction(
- { id: 'mem::summarize', description: 'Generate end-of-session summary' },
+ { id: "mem::summarize", description: "Generate end-of-session summary" },
async (data: { sessionId: string }) => {
- const ctx = getContext()
+ const ctx = getContext();
+ const startMs = Date.now();
- const session = await kv.get<Session>(KV.sessions, data.sessionId)
+ const session = await kv.get<Session>(KV.sessions, data.sessionId);
if (!session) {
- ctx.logger.warn('Session not found for summarize', { sessionId: data.sessionId })
- return { success: false, error: 'session_not_found' }
+ ctx.logger.warn("Session not found for summarize", {
+ sessionId: data.sessionId,
+ });
+ return { success: false, error: "session_not_found" };
}
- const observations = await kv.list<CompressedObservation>(KV.observations(data.sessionId))
- const compressed = observations.filter((o) => o.title)
+ const observations = await kv.list<CompressedObservation>(
+ KV.observations(data.sessionId),
+ );
+ const compressed = observations.filter((o) => o.title);
if (compressed.length === 0) {
- ctx.logger.info('No observations to summarize', { sessionId: data.sessionId })
- return { success: false, error: 'no_observations' }
+ ctx.logger.info("No observations to summarize", {
+ sessionId: data.sessionId,
+ });
+ return { success: false, error: "no_observations" };
}
try {
- const prompt = buildSummaryPrompt(compressed)
- const response = await provider.summarize(SUMMARY_SYSTEM, prompt)
- const summary = parseSummaryXml(response, data.sessionId, session.project, compressed.length)
+ const prompt = buildSummaryPrompt(compressed);
+ const response = await provider.summarize(SUMMARY_SYSTEM, prompt);
+ const summary = parseSummaryXml(
+ response,
+ data.sessionId,
+ session.project,
+ compressed.length,
+ );
if (!summary) {
- ctx.logger.warn('Failed to parse summary XML', { sessionId: data.sessionId })
- return { success: false, error: 'parse_failed' }
+ const latencyMs = Date.now() - startMs;
+ if (metricsStore) {
+ await metricsStore.record("mem::summarize", latencyMs, false);
+ }
+ ctx.logger.warn("Failed to parse summary XML", {
+ sessionId: data.sessionId,
+ });
+ return { success: false, error: "parse_failed" };
}
- await kv.set(KV.summaries, data.sessionId, summary)
+ const summaryForValidation = {
+ title: summary.title,
+ narrative: summary.narrative,
+ keyDecisions: summary.keyDecisions,
+ filesModified: summary.filesModified,
+ concepts: summary.concepts,
+ };
+ const validation = validateOutput(
+ SummaryOutputSchema,
+ summaryForValidation,
+ "mem::summarize",
+ );
+
+ const qualityScore = scoreSummary(summaryForValidation);
+
+ await kv.set(KV.summaries, data.sessionId, summary);
- ctx.logger.info('Session summarized', {
+ const latencyMs = Date.now() - startMs;
+ if (metricsStore) {
+ await metricsStore.record(
+ "mem::summarize",
+ latencyMs,
+ true,
+ qualityScore,
+ );
+ }
+
+ ctx.logger.info("Session summarized", {
sessionId: data.sessionId,
title: summary.title,
decisions: summary.keyDecisions.length,
- })
+ qualityScore,
+ valid: validation.valid,
+ });
- return { success: true, summary }
+ return { success: true, summary, qualityScore };
} catch (err) {
- const msg = err instanceof Error ? err.message : String(err)
- ctx.logger.error('Summarize failed', { sessionId: data.sessionId, error: msg })
- return { success: false, error: msg }
+ const msg = err instanceof Error ? err.message : String(err);
+ const latencyMs = Date.now() - startMs;
+ if (metricsStore) {
+ await metricsStore.record("mem::summarize", latencyMs, false);
+ }
+ ctx.logger.error("Summarize failed", {
+ sessionId: data.sessionId,
+ error: msg,
+ });
+ return { success: false, error: msg };
}
- }
- )
+ },
+ );
}
diff --git a/src/health/monitor.ts b/src/health/monitor.ts
new file mode 100644
index 00000000..5cd6e0ba
--- /dev/null
+++ b/src/health/monitor.ts
@@ -0,0 +1,77 @@
+import type { ISdk } from "iii-sdk";
+import type { HealthSnapshot } from "../types.js";
+import type { StateKV } from "../state/kv.js";
+import { KV } from "../state/schema.js";
+import { evaluateHealth } from "./thresholds.js";
+
+let connectionState = "connected";
+
+export function registerHealthMonitor(
+ sdk: ISdk,
+ kv: StateKV,
+): { stop: () => void } {
+ if (typeof sdk.on === "function") {
+ sdk.on("connection_state", (state: string) => {
+ connectionState = state;
+ });
+ }
+
+ async function collectHealth(): Promise<HealthSnapshot> {
+ const mem = process.memoryUsage();
+ const cpu = process.cpuUsage();
+ const uptime = process.uptime();
+
+ const startMark = performance.now();
+ await new Promise((resolve) => setImmediate(resolve));
+ const eventLoopLagMs = performance.now() - startMark;
+
+ let workers: HealthSnapshot["workers"] = [];
+ try {
+ const result = await sdk.trigger<unknown, { workers?: HealthSnapshot["workers"] }>(
+ "engine::workers::list",
+ {},
+ );
+ if (result?.workers) workers = result.workers;
+ } catch {}
+
+ const snapshot: HealthSnapshot = {
+ connectionState,
+ workers,
+ memory: {
+ heapUsed: mem.heapUsed,
+ heapTotal: mem.heapTotal,
+ rss: mem.rss,
+ external: mem.external,
+ },
+ cpu: {
+ userMicros: cpu.user,
+ systemMicros: cpu.system,
+ percent: 0,
+ },
+ eventLoopLagMs,
+ uptimeSeconds: uptime,
+ status: "healthy",
+ alerts: [],
+ };
+
+ const evaluated = evaluateHealth(snapshot);
+ snapshot.status = evaluated.status;
+ snapshot.alerts = evaluated.alerts;
+
+ await kv.set(KV.health, "latest", snapshot).catch(() => {});
+ return snapshot;
+ }
+
+ collectHealth().catch(() => {});
+ const interval = setInterval(() => {
+ collectHealth().catch(() => {});
+ }, 30_000);
+
+ return {
+ stop: () => clearInterval(interval),
+ };
+}
+
+export async function getLatestHealth(kv: StateKV): Promise<HealthSnapshot | null> {
+ return kv.get<HealthSnapshot>(KV.health, "latest");
+}
diff --git a/src/health/recovery.ts b/src/health/recovery.ts
new file mode 100644
index 00000000..34be8bb3
--- /dev/null
+++ b/src/health/recovery.ts
@@ -0,0 +1,22 @@
+import type { HookPayload } from "../types.js";
+
+const MAX_QUEUE_SIZE = 500;
+
+export class ObservationQueue {
+ private queue: HookPayload[] = [];
+
+ enqueue(payload: HookPayload): boolean {
+ if (this.queue.length >= MAX_QUEUE_SIZE) return false;
+ this.queue.push(payload);
+ return true;
+ }
+
+ drain(): HookPayload[] {
+ const items = this.queue.splice(0);
+ return items;
+ }
+
+ get size(): number {
+ return this.queue.length;
+ }
+}
diff --git a/src/health/thresholds.ts b/src/health/thresholds.ts
new file mode 100644
index 00000000..373b4264
--- /dev/null
+++ b/src/health/thresholds.ts
@@ -0,0 +1,67 @@
+import type { HealthSnapshot } from "../types.js";
+
+interface ThresholdConfig {
+ eventLoopLagWarnMs: number;
+ eventLoopLagCriticalMs: number;
+ cpuWarnPercent: number;
+ cpuCriticalPercent: number;
+ memoryWarnPercent: number;
+ memoryCriticalPercent: number;
+}
+
+const DEFAULTS: ThresholdConfig = {
+ eventLoopLagWarnMs: 100,
+ eventLoopLagCriticalMs: 500,
+ cpuWarnPercent: 80,
+ cpuCriticalPercent: 90,
+ memoryWarnPercent: 80,
+ memoryCriticalPercent: 95,
+};
+
+export function evaluateHealth(
+ snapshot: HealthSnapshot,
+ config: Partial<ThresholdConfig> = {},
+): { status: "healthy" | "degraded" | "critical"; alerts: string[] } {
+ const cfg = { ...DEFAULTS, ...config };
+ const alerts: string[] = [];
+ let critical = false;
+ let degraded = false;
+
+ if (snapshot.connectionState === "disconnected" || snapshot.connectionState === "failed") {
+ alerts.push(`connection_${snapshot.connectionState}`);
+ critical = true;
+ } else if (snapshot.connectionState === "reconnecting") {
+ alerts.push("connection_reconnecting");
+ degraded = true;
+ }
+
+ if (snapshot.eventLoopLagMs > cfg.eventLoopLagCriticalMs) {
+ alerts.push(`event_loop_lag_critical_${Math.round(snapshot.eventLoopLagMs)}ms`);
+ critical = true;
+ } else if (snapshot.eventLoopLagMs > cfg.eventLoopLagWarnMs) {
+ alerts.push(`event_loop_lag_warn_${Math.round(snapshot.eventLoopLagMs)}ms`);
+ degraded = true;
+ }
+
+ if (snapshot.cpu.percent > cfg.cpuCriticalPercent) {
+ alerts.push(`cpu_critical_${Math.round(snapshot.cpu.percent)}%`);
+ critical = true;
+ } else if (snapshot.cpu.percent > cfg.cpuWarnPercent) {
+ alerts.push(`cpu_warn_${Math.round(snapshot.cpu.percent)}%`);
+ degraded = true;
+ }
+
+ const memPercent = snapshot.memory.heapTotal > 0
+ ? (snapshot.memory.rss / snapshot.memory.heapTotal) * 100
+ : 0;
+ if (memPercent > cfg.memoryCriticalPercent) {
+ alerts.push(`memory_critical_${Math.round(memPercent)}%`);
+ critical = true;
+ } else if (memPercent > cfg.memoryWarnPercent) {
+ alerts.push(`memory_warn_${Math.round(memPercent)}%`);
+ degraded = true;
+ }
+
+ const status = critical ? "critical" : degraded ? "degraded" : "healthy";
+ return { status, alerts };
+}
diff --git a/src/index.ts b/src/index.ts
index 0ba06f60..bbf73ec7 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -13,15 +13,21 @@ import { registerFileIndexFunction } from "./functions/file-index.js";
import { registerConsolidateFunction } from "./functions/consolidate.js";
import { registerPatternsFunction } from "./functions/patterns.js";
import { registerRememberFunction } from "./functions/remember.js";
+import { registerEvictFunction } from "./functions/evict.js";
import { registerApiTriggers } from "./triggers/api.js";
import { registerEventTriggers } from "./triggers/events.js";
import { registerMcpEndpoints } from "./mcp/server.js";
+import { MetricsStore } from "./eval/metrics-store.js";
+import { DedupMap } from "./functions/dedup.js";
+import { ObservationQueue } from "./health/recovery.js";
+import { registerHealthMonitor, getLatestHealth } from "./health/monitor.js";
+import { initMetrics, OTEL_CONFIG } from "./telemetry/setup.js";
async function main() {
const config = loadConfig();
const provider = createProvider(config.provider);
- console.log(`[agentmemory] Starting worker...`);
+ console.log(`[agentmemory] Starting worker v0.3.0...`);
console.log(`[agentmemory] Engine: ${config.engineUrl}`);
console.log(
`[agentmemory] Provider: ${config.provider.provider} (${config.provider.model})`,
@@ -33,27 +39,42 @@ async function main() {
const sdk = init(config.engineUrl, {
workerName: "agentmemory",
+ otel: {
+ serviceName: OTEL_CONFIG.serviceName,
+ serviceVersion: OTEL_CONFIG.serviceVersion,
+ metricsExportIntervalMs: OTEL_CONFIG.metricsExportIntervalMs,
+ },
});
const kv = new StateKV(sdk);
const secret = getEnvVar("AGENTMEMORY_SECRET");
+ const metricsStore = new MetricsStore(kv);
+ const dedupMap = new DedupMap();
+ const observationQueue = new ObservationQueue();
+
+ const { counters, histograms } = initMetrics(
+ typeof sdk.getMeter === "function" ? sdk.getMeter.bind(sdk) : undefined,
+ );
registerPrivacyFunction(sdk);
- registerObserveFunction(sdk, kv);
- registerCompressFunction(sdk, kv, provider);
+ registerObserveFunction(sdk, kv, dedupMap);
+ registerCompressFunction(sdk, kv, provider, metricsStore);
registerSearchFunction(sdk, kv);
registerContextFunction(sdk, kv, config.tokenBudget);
- registerSummarizeFunction(sdk, kv, provider);
+ registerSummarizeFunction(sdk, kv, provider, metricsStore);
registerMigrateFunction(sdk, kv);
registerFileIndexFunction(sdk, kv);
registerConsolidateFunction(sdk, kv, provider);
registerPatternsFunction(sdk, kv);
registerRememberFunction(sdk, kv);
+ registerEvictFunction(sdk, kv);
- registerApiTriggers(sdk, kv, secret);
+ registerApiTriggers(sdk, kv, secret, metricsStore, provider);
registerEventTriggers(sdk, kv);
registerMcpEndpoints(sdk, kv, secret);
+ const healthMonitor = registerHealthMonitor(sdk, kv);
+
const indexCount = await rebuildIndex(kv).catch(() => 0);
if (indexCount > 0) {
console.log(`[agentmemory] Search index: ${indexCount} observations`);
@@ -74,11 +95,12 @@ async function main() {
console.log(` POST /agentmemory/file-context - File history context`);
console.log(` POST /agentmemory/consolidate - Consolidate memories`);
console.log(` POST /agentmemory/patterns - Detect patterns`);
+ console.log(` POST /agentmemory/evict - Evict stale memories`);
console.log(` GET /agentmemory/sessions - List sessions`);
console.log(
` GET /agentmemory/observations - Get session observations`,
);
- console.log(` GET /agentmemory/health - Health check`);
+ console.log(` GET /agentmemory/health - Health + metrics`);
console.log(` GET /agentmemory/viewer - Web viewer`);
console.log(
` POST /agentmemory/generate-rules - Generate rules from patterns`,
@@ -89,6 +111,8 @@ async function main() {
const shutdown = async () => {
console.log(`\n[agentmemory] Shutting down...`);
+ healthMonitor.stop();
+ dedupMap.stop();
await sdk.shutdown();
process.exit(0);
};
diff --git a/src/providers/circuit-breaker.ts b/src/providers/circuit-breaker.ts
new file mode 100644
index 00000000..3b24acf3
--- /dev/null
+++ b/src/providers/circuit-breaker.ts
@@ -0,0 +1,60 @@
+import type { CircuitBreakerState } from "../types.js";
+
+const FAILURE_THRESHOLD = 3;
+const FAILURE_WINDOW_MS = 60_000;
+const RECOVERY_TIMEOUT_MS = 30_000;
+
+export class CircuitBreaker {
+ private state: "closed" | "open" | "half-open" = "closed";
+ private failures = 0;
+ private lastFailureAt: number | null = null;
+ private openedAt: number | null = null;
+
+ get isAllowed(): boolean {
+ if (this.state === "closed") return true;
+ if (this.state === "open") {
+ if (this.openedAt && Date.now() - this.openedAt >= RECOVERY_TIMEOUT_MS) {
+ this.state = "half-open";
+ return true;
+ }
+ return false;
+ }
+ return true;
+ }
+
+ recordSuccess(): void {
+ if (this.state === "half-open") {
+ this.state = "closed";
+ this.failures = 0;
+ this.lastFailureAt = null;
+ this.openedAt = null;
+ }
+ }
+
+ recordFailure(): void {
+ const now = Date.now();
+ if (this.state === "half-open") {
+ this.state = "open";
+ this.openedAt = now;
+ return;
+ }
+ if (this.lastFailureAt && now - this.lastFailureAt > FAILURE_WINDOW_MS) {
+ this.failures = 0;
+ }
+ this.failures += 1;
+ this.lastFailureAt = now;
+ if (this.failures >= FAILURE_THRESHOLD) {
+ this.state = "open";
+ this.openedAt = now;
+ }
+ }
+
+ getState(): CircuitBreakerState {
+ return {
+ state: this.state,
+ failures: this.failures,
+ lastFailureAt: this.lastFailureAt,
+ openedAt: this.openedAt,
+ };
+ }
+}
diff --git a/src/providers/index.ts b/src/providers/index.ts
index 06589149..8649b320 100644
--- a/src/providers/index.ts
+++ b/src/providers/index.ts
@@ -1,29 +1,40 @@
-import type { MemoryProvider, ProviderConfig } from '../types.js'
-import { AgentSDKProvider } from './agent-sdk.js'
-import { AnthropicProvider } from './anthropic.js'
-import { OpenRouterProvider } from './openrouter.js'
-import { getEnvVar } from '../config.js'
+import type { MemoryProvider, ProviderConfig } from "../types.js";
+import { AgentSDKProvider } from "./agent-sdk.js";
+import { AnthropicProvider } from "./anthropic.js";
+import { OpenRouterProvider } from "./openrouter.js";
+import { ResilientProvider } from "./resilient.js";
+import { getEnvVar } from "../config.js";
-export function createProvider(config: ProviderConfig): MemoryProvider {
+export function createProvider(config: ProviderConfig): ResilientProvider {
+ let base: MemoryProvider;
switch (config.provider) {
- case 'anthropic':
- return new AnthropicProvider(getEnvVar('ANTHROPIC_API_KEY')!, config.model, config.maxTokens)
- case 'gemini':
- return new OpenRouterProvider(
- getEnvVar('GEMINI_API_KEY')!,
+ case "anthropic":
+ base = new AnthropicProvider(
+ getEnvVar("ANTHROPIC_API_KEY")!,
config.model,
config.maxTokens,
- 'https://generativelanguage.googleapis.com/v1beta/openai/chat/completions'
- )
- case 'openrouter':
- return new OpenRouterProvider(
- getEnvVar('OPENROUTER_API_KEY')!,
+ );
+ break;
+ case "gemini":
+ base = new OpenRouterProvider(
+ getEnvVar("GEMINI_API_KEY")!,
config.model,
config.maxTokens,
- 'https://openrouter.ai/api/v1/chat/completions'
- )
- case 'agent-sdk':
+ "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions",
+ );
+ break;
+ case "openrouter":
+ base = new OpenRouterProvider(
+ getEnvVar("OPENROUTER_API_KEY")!,
+ config.model,
+ config.maxTokens,
+ "https://openrouter.ai/api/v1/chat/completions",
+ );
+ break;
+ case "agent-sdk":
default:
- return new AgentSDKProvider()
+ base = new AgentSDKProvider();
+ break;
}
+ return new ResilientProvider(base);
}
diff --git a/src/providers/resilient.ts b/src/providers/resilient.ts
new file mode 100644
index 00000000..d7e2a02a
--- /dev/null
+++ b/src/providers/resilient.ts
@@ -0,0 +1,43 @@
+import type { MemoryProvider, CircuitBreakerState } from "../types.js";
+import { CircuitBreaker } from "./circuit-breaker.js";
+
+export class ResilientProvider implements MemoryProvider {
+ private breaker = new CircuitBreaker();
+ name: string;
+
+ constructor(private inner: MemoryProvider) {
+ this.name = `resilient(${inner.name})`;
+ }
+
+ async compress(systemPrompt: string, userPrompt: string): Promise<string> {
+ if (!this.breaker.isAllowed) {
+ throw new Error("circuit_breaker_open");
+ }
+ try {
+ const result = await this.inner.compress(systemPrompt, userPrompt);
+ this.breaker.recordSuccess();
+ return result;
+ } catch (err) {
+ this.breaker.recordFailure();
+ throw err;
+ }
+ }
+
+ async summarize(systemPrompt: string, userPrompt: string): Promise<string> {
+ if (!this.breaker.isAllowed) {
+ throw new Error("circuit_breaker_open");
+ }
+ try {
+ const result = await this.inner.summarize(systemPrompt, userPrompt);
+ this.breaker.recordSuccess();
+ return result;
+ } catch (err) {
+ this.breaker.recordFailure();
+ throw err;
+ }
+ }
+
+ get circuitState(): CircuitBreakerState {
+ return this.breaker.getState();
+ }
+}
diff --git a/src/state/schema.ts b/src/state/schema.ts
index 236ba915..0fc51398 100644
--- a/src/state/schema.ts
+++ b/src/state/schema.ts
@@ -4,6 +4,8 @@ export const KV = {
memories: "mem:memories",
summaries: "mem:summaries",
config: "mem:config",
+ metrics: "mem:metrics",
+ health: "mem:health",
} as const;
export const STREAM = {
diff --git a/src/state/search-index.ts b/src/state/search-index.ts
index bc40255b..e667c16b 100644
--- a/src/state/search-index.ts
+++ b/src/state/search-index.ts
@@ -3,17 +3,33 @@ import type { CompressedObservation } from "../types.js";
interface IndexEntry {
obsId: string;
sessionId: string;
+ termCount: number;
}
export class SearchIndex {
private entries: Map<string, IndexEntry> = new Map();
private invertedIndex: Map<string, Set<string>> = new Map();
+ private docTermCounts: Map<string, Map<string, number>> = new Map();
+ private totalDocLength = 0;
+
+ private readonly k1 = 1.2;
+ private readonly b = 0.75;
add(obs: CompressedObservation): void {
const terms = this.extractTerms(obs);
- this.entries.set(obs.id, { obsId: obs.id, sessionId: obs.sessionId });
+ const termFreq = new Map<string, number>();
+ let termCount = 0;
for (const term of terms) {
+ termFreq.set(term, (termFreq.get(term) || 0) + 1);
+ termCount++;
+ }
+
+ this.entries.set(obs.id, { obsId: obs.id, sessionId: obs.sessionId, termCount });
+ this.docTermCounts.set(obs.id, termFreq);
+ this.totalDocLength += termCount;
+
+ for (const term of termFreq.keys()) {
if (!this.invertedIndex.has(term)) {
this.invertedIndex.set(term, new Set());
}
@@ -28,19 +44,44 @@ export class SearchIndex {
const queryTerms = this.tokenize(query.toLowerCase());
if (queryTerms.length === 0) return [];
+ const N = this.entries.size;
+ if (N === 0) return [];
+ const avgDocLen = this.totalDocLength / N;
+
const scores = new Map<string, number>();
for (const term of queryTerms) {
- const matches = this.invertedIndex.get(term);
- if (!matches) continue;
- for (const obsId of matches) {
- scores.set(obsId, (scores.get(obsId) || 0) + 1);
+ const matchingDocs = this.invertedIndex.get(term);
+ if (!matchingDocs) continue;
+
+ const df = matchingDocs.size;
+ const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+
+ for (const obsId of matchingDocs) {
+ const entry = this.entries.get(obsId)!;
+ const docTerms = this.docTermCounts.get(obsId);
+ const tf = docTerms?.get(term) || 0;
+ const docLen = entry.termCount;
+
+ const numerator = tf * (this.k1 + 1);
+ const denominator = tf + this.k1 * (1 - this.b + this.b * (docLen / avgDocLen));
+ const bm25Score = idf * (numerator / denominator);
+
+ scores.set(obsId, (scores.get(obsId) || 0) + bm25Score);
}
for (const [indexTerm, obsIds] of this.invertedIndex) {
if (indexTerm !== term && indexTerm.startsWith(term)) {
+ const prefixDf = obsIds.size;
+ const prefixIdf = Math.log((N - prefixDf + 0.5) / (prefixDf + 0.5) + 1) * 0.5;
for (const obsId of obsIds) {
- scores.set(obsId, (scores.get(obsId) || 0) + 0.5);
+ const entry = this.entries.get(obsId)!;
+ const docTerms = this.docTermCounts.get(obsId);
+ const tf = docTerms?.get(indexTerm) || 0;
+ const docLen = entry.termCount;
+ const numerator = tf * (this.k1 + 1);
+ const denominator = tf + this.k1 * (1 - this.b + this.b * (docLen / avgDocLen));
+ scores.set(obsId, (scores.get(obsId) || 0) + prefixIdf * (numerator / denominator));
}
}
}
@@ -49,11 +90,7 @@ export class SearchIndex {
return Array.from(scores.entries())
.map(([obsId, score]) => {
const entry = this.entries.get(obsId)!;
- return {
- obsId,
- sessionId: entry.sessionId,
- score: score / queryTerms.length,
- };
+ return { obsId, sessionId: entry.sessionId, score };
})
.sort((a, b) => b.score - a.score)
.slice(0, limit);
@@ -66,9 +103,11 @@ export class SearchIndex {
clear(): void {
this.entries.clear();
this.invertedIndex.clear();
+ this.docTermCounts.clear();
+ this.totalDocLength = 0;
}
- private extractTerms(obs: CompressedObservation): Set<string> {
+ private extractTerms(obs: CompressedObservation): string[] {
const parts = [
obs.title,
obs.subtitle || "",
@@ -78,7 +117,7 @@ export class SearchIndex {
...obs.files,
obs.type,
];
- return new Set(this.tokenize(parts.join(" ").toLowerCase()));
+ return this.tokenize(parts.join(" ").toLowerCase());
}
private tokenize(text: string): string[] {
diff --git a/src/telemetry/setup.ts b/src/telemetry/setup.ts
new file mode 100644
index 00000000..bc442dd2
--- /dev/null
+++ b/src/telemetry/setup.ts
@@ -0,0 +1,96 @@
+interface OtelConfig {
+ serviceName: string;
+ serviceVersion: string;
+ metricsExportIntervalMs: number;
+}
+
+export const OTEL_CONFIG: OtelConfig = {
+ serviceName: "agentmemory",
+ serviceVersion: "0.3.0",
+ metricsExportIntervalMs: 30_000,
+};
+
+interface Counters {
+ observationsTotal: { add: (n: number) => void };
+ compressionSuccess: { add: (n: number) => void };
+ compressionFailure: { add: (n: number) => void };
+ searchTotal: { add: (n: number) => void };
+ dedupSkipped: { add: (n: number) => void };
+ evictionTotal: { add: (n: number) => void };
+ circuitBreakerOpen: { add: (n: number) => void };
+}
+
+interface Histograms {
+ compressionLatency: { record: (v: number) => void };
+ searchLatency: { record: (v: number) => void };
+ contextTokens: { record: (v: number) => void };
+ qualityScore: { record: (v: number) => void };
+}
+
+let counters: Counters | null = null;
+let histograms: Histograms | null = null;
+
+function noopCounter() {
+ return { add: () => {} };
+}
+
+function noopHistogram() {
+ return { record: () => {} };
+}
+
+export function initMetrics(getMeter?: (name: string) => {
+ createCounter: (name: string) => { add: (n: number) => void };
+ createHistogram: (name: string) => { record: (v: number) => void };
+}): { counters: Counters; histograms: Histograms } {
+ if (getMeter) {
+ const meter = getMeter("agentmemory");
+ counters = {
+ observationsTotal: meter.createCounter("observations.total"),
+ compressionSuccess: meter.createCounter("compression.success"),
+ compressionFailure: meter.createCounter("compression.failure"),
+ searchTotal: meter.createCounter("search.total"),
+ dedupSkipped: meter.createCounter("dedup.skipped"),
+ evictionTotal: meter.createCounter("eviction.total"),
+ circuitBreakerOpen: meter.createCounter("circuit_breaker.open"),
+ };
+ histograms = {
+ compressionLatency: meter.createHistogram("compression.latency_ms"),
+ searchLatency: meter.createHistogram("search.latency_ms"),
+ contextTokens: meter.createHistogram("context.tokens"),
+ qualityScore: meter.createHistogram("quality.score"),
+ };
+ } else {
+ counters = {
+ observationsTotal: noopCounter(),
+ compressionSuccess: noopCounter(),
+ compressionFailure: noopCounter(),
+ searchTotal: noopCounter(),
+ dedupSkipped: noopCounter(),
+ evictionTotal: noopCounter(),
+ circuitBreakerOpen: noopCounter(),
+ };
+ histograms = {
+ compressionLatency: noopHistogram(),
+ searchLatency: noopHistogram(),
+ contextTokens: noopHistogram(),
+ qualityScore: noopHistogram(),
+ };
+ }
+ return { counters, histograms };
+}
+
+export function getCounters(): Counters {
+ if (!counters) {
+ const { counters: c } = initMetrics();
+ return c;
+ }
+ return counters;
+}
+
+export function getHistograms(): Histograms {
+ if (!histograms) {
+ const { histograms: h } = initMetrics();
+ return h;
+ }
+ return histograms;
+}
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index 8cc9dba9..3ed5f059 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -5,6 +5,9 @@ import { StateKV } from "../state/kv.js";
import { readFileSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
+import { getLatestHealth } from "../health/monitor.js";
+import type { MetricsStore } from "../eval/metrics-store.js";
+import type { ResilientProvider } from "../providers/resilient.js";
type Response = {
status_code: number;
@@ -31,13 +34,34 @@ export function registerApiTriggers(
sdk: ISdk,
kv: StateKV,
secret?: string,
+ metricsStore?: MetricsStore,
+ provider?: ResilientProvider | { circuitState?: unknown },
): void {
sdk.registerFunction(
{ id: "api::health" },
- async (): Promise<Response> => ({
- status_code: 200,
- body: { status: "ok", service: "agentmemory", version: "0.2.0" },
- }),
+ async (): Promise<Response> => {
+ const health = await getLatestHealth(kv);
+ const functionMetrics = metricsStore
+ ? await metricsStore.getAll()
+ : [];
+ const circuitBreaker =
+ provider && "circuitState" in provider ? provider.circuitState : null;
+
+ const status = health?.status || "healthy";
+ const statusCode = status === "critical" ? 503 : 200;
+
+ return {
+ status_code: statusCode,
+ body: {
+ status,
+ service: "agentmemory",
+ version: "0.3.0",
+ health: health || null,
+ functionMetrics,
+ circuitBreaker,
+ },
+ };
+ },
);
sdk.registerTrigger({
type: "http",
@@ -330,6 +354,24 @@ export function registerApiTriggers(
config: { api_path: "/agentmemory/migrate", http_method: "POST" },
});
+ sdk.registerFunction(
+ { id: "api::evict" },
+ async (
+ req: ApiRequest<{ dryRun?: boolean }>,
+ ): Promise<Response> => {
+ const authErr = checkAuth(req, secret);
+ if (authErr) return authErr;
+ const dryRun = req.query_params?.["dryRun"] === "true" || req.body?.dryRun;
+ const result = await sdk.trigger("mem::evict", { dryRun });
+ return { status_code: 200, body: result };
+ },
+ );
+ sdk.registerTrigger({
+ type: "http",
+ function_id: "api::evict",
+ config: { api_path: "/agentmemory/evict", http_method: "POST" },
+ });
+
sdk.registerFunction({ id: "api::viewer" }, async (): Promise<Response> => {
const headers = {
"Content-Type": "text/html",
diff --git a/src/types.ts b/src/types.ts
index a405cb88..de0be266 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -138,3 +138,43 @@ export interface ContextBlock {
tokens: number;
recency: number;
}
+
+export interface EvalResult {
+ valid: boolean;
+ errors: string[];
+ qualityScore: number;
+ latencyMs: number;
+ functionId: string;
+}
+
+export interface FunctionMetrics {
+ functionId: string;
+ totalCalls: number;
+ successCount: number;
+ failureCount: number;
+ avgLatencyMs: number;
+ avgQualityScore: number;
+}
+
+export interface HealthSnapshot {
+ connectionState: string;
+ workers: Array<{ id: string; name: string; status: string }>;
+ memory: {
+ heapUsed: number;
+ heapTotal: number;
+ rss: number;
+ external: number;
+ };
+ cpu: { userMicros: number; systemMicros: number; percent: number };
+ eventLoopLagMs: number;
+ uptimeSeconds: number;
+ status: "healthy" | "degraded" | "critical";
+ alerts: string[];
+}
+
+export interface CircuitBreakerState {
+ state: "closed" | "open" | "half-open";
+ failures: number;
+ lastFailureAt: number | null;
+ openedAt: number | null;
+}
diff --git a/test/circuit-breaker.test.ts b/test/circuit-breaker.test.ts
new file mode 100644
index 00000000..a4e32a30
--- /dev/null
+++ b/test/circuit-breaker.test.ts
@@ -0,0 +1,107 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { CircuitBreaker } from "../src/providers/circuit-breaker.js";
+
+describe("CircuitBreaker", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("starts in closed state", () => {
+ const cb = new CircuitBreaker();
+ expect(cb.getState().state).toBe("closed");
+ expect(cb.isAllowed).toBe(true);
+ });
+
+ it("stays closed after fewer than 3 failures", () => {
+ const cb = new CircuitBreaker();
+ cb.recordFailure();
+ cb.recordFailure();
+ expect(cb.getState().state).toBe("closed");
+ expect(cb.isAllowed).toBe(true);
+ });
+
+ it("opens after 3 failures within the window", () => {
+ const cb = new CircuitBreaker();
+ cb.recordFailure();
+ cb.recordFailure();
+ cb.recordFailure();
+ expect(cb.getState().state).toBe("open");
+ expect(cb.isAllowed).toBe(false);
+ });
+
+ it("resets failure count when failures are outside the window", () => {
+ const cb = new CircuitBreaker();
+ cb.recordFailure();
+ cb.recordFailure();
+ vi.advanceTimersByTime(61_000);
+ cb.recordFailure();
+ expect(cb.getState().state).toBe("closed");
+ expect(cb.getState().failures).toBe(1);
+ });
+
+ it("transitions to half-open after recovery timeout", () => {
+ const cb = new CircuitBreaker();
+ cb.recordFailure();
+ cb.recordFailure();
+ cb.recordFailure();
+ expect(cb.isAllowed).toBe(false);
+ vi.advanceTimersByTime(30_000);
+ expect(cb.isAllowed).toBe(true);
+ expect(cb.getState().state).toBe("half-open");
+ });
+
+ it("closes on success in half-open state", () => {
+ const cb = new CircuitBreaker();
+ cb.recordFailure();
+ cb.recordFailure();
+ cb.recordFailure();
+ vi.advanceTimersByTime(30_000);
+ cb.isAllowed;
+ cb.recordSuccess();
+ expect(cb.getState().state).toBe("closed");
+ expect(cb.getState().failures).toBe(0);
+ expect(cb.getState().lastFailureAt).toBeNull();
+ });
+
+ it("reopens on failure in half-open state", () => {
+ const cb = new CircuitBreaker();
+ cb.recordFailure();
+ cb.recordFailure();
+ cb.recordFailure();
+ vi.advanceTimersByTime(30_000);
+ cb.isAllowed;
+ cb.recordFailure();
+ expect(cb.getState().state).toBe("open");
+ });
+
+ it("records lastFailureAt timestamp", () => {
+ const cb = new CircuitBreaker();
+ vi.setSystemTime(new Date("2026-01-15T10:00:00Z"));
+ cb.recordFailure();
+ expect(cb.getState().lastFailureAt).toBe(
+ new Date("2026-01-15T10:00:00Z").getTime(),
+ );
+ });
+
+ it("records openedAt timestamp", () => {
+ const cb = new CircuitBreaker();
+ vi.setSystemTime(new Date("2026-01-15T10:00:00Z"));
+ cb.recordFailure();
+ cb.recordFailure();
+ cb.recordFailure();
+ expect(cb.getState().openedAt).toBe(
+ new Date("2026-01-15T10:00:00Z").getTime(),
+ );
+ });
+
+ it("success in closed state is a no-op", () => {
+ const cb = new CircuitBreaker();
+ cb.recordSuccess();
+ expect(cb.getState().state).toBe("closed");
+ expect(cb.getState().failures).toBe(0);
+ });
+});
diff --git a/test/eval.test.ts b/test/eval.test.ts
new file mode 100644
index 00000000..f1070e1a
--- /dev/null
+++ b/test/eval.test.ts
@@ -0,0 +1,300 @@
+import { describe, it, expect } from "vitest";
+import {
+ ObserveInputSchema,
+ CompressOutputSchema,
+ SummaryOutputSchema,
+ SearchInputSchema,
+ ContextInputSchema,
+ RememberInputSchema,
+} from "../src/eval/schemas.js";
+import { validateInput, validateOutput } from "../src/eval/validator.js";
+import {
+ scoreCompression,
+ scoreSummary,
+ scoreContextRelevance,
+} from "../src/eval/quality.js";
+
+describe("Zod Schemas", () => {
+ describe("ObserveInputSchema", () => {
+ it("accepts valid input", () => {
+ const result = ObserveInputSchema.safeParse({
+ hookType: "post_tool_use",
+ sessionId: "ses_abc",
+ project: "my-project",
+ cwd: "/home/user",
+ timestamp: "2026-01-01T00:00:00Z",
+ data: { tool_name: "Read" },
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it("rejects missing sessionId", () => {
+ const result = ObserveInputSchema.safeParse({
+ hookType: "post_tool_use",
+ project: "my-project",
+ cwd: "/home/user",
+ timestamp: "2026-01-01T00:00:00Z",
+ data: {},
+ });
+ expect(result.success).toBe(false);
+ });
+
+ it("rejects invalid hookType", () => {
+ const result = ObserveInputSchema.safeParse({
+ hookType: "invalid_hook",
+ sessionId: "ses_abc",
+ project: "my-project",
+ cwd: "/home/user",
+ timestamp: "2026-01-01T00:00:00Z",
+ data: {},
+ });
+ expect(result.success).toBe(false);
+ });
+ });
+
+ describe("CompressOutputSchema", () => {
+ it("accepts valid output", () => {
+ const result = CompressOutputSchema.safeParse({
+ type: "file_edit",
+ title: "Edit auth module",
+ facts: ["Added JWT validation"],
+ narrative: "Modified the auth middleware to validate tokens",
+ concepts: ["auth"],
+ files: ["src/auth.ts"],
+ importance: 7,
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it("rejects empty facts array", () => {
+ const result = CompressOutputSchema.safeParse({
+ type: "file_edit",
+ title: "Edit auth module",
+ facts: [],
+ narrative: "Modified the auth middleware to validate tokens",
+ concepts: [],
+ files: [],
+ importance: 5,
+ });
+ expect(result.success).toBe(false);
+ });
+
+ it("rejects title over 120 chars", () => {
+ const result = CompressOutputSchema.safeParse({
+ type: "file_edit",
+ title: "x".repeat(121),
+ facts: ["fact"],
+ narrative: "A narrative that is long enough",
+ concepts: [],
+ files: [],
+ importance: 5,
+ });
+ expect(result.success).toBe(false);
+ });
+
+ it("rejects importance outside 1-10", () => {
+ const result = CompressOutputSchema.safeParse({
+ type: "file_edit",
+ title: "Test",
+ facts: ["fact"],
+ narrative: "A valid narrative here",
+ concepts: [],
+ files: [],
+ importance: 11,
+ });
+ expect(result.success).toBe(false);
+ });
+
+ it("rejects narrative under 10 chars", () => {
+ const result = CompressOutputSchema.safeParse({
+ type: "file_edit",
+ title: "Test",
+ facts: ["fact"],
+ narrative: "short",
+ concepts: [],
+ files: [],
+ importance: 5,
+ });
+ expect(result.success).toBe(false);
+ });
+ });
+
+ describe("SummaryOutputSchema", () => {
+ it("accepts valid summary", () => {
+ const result = SummaryOutputSchema.safeParse({
+ title: "Session Summary",
+ narrative: "This session focused on implementing authentication features and fixing bugs",
+ keyDecisions: ["Use JWT"],
+ filesModified: ["auth.ts"],
+ concepts: ["auth"],
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it("rejects short narrative", () => {
+ const result = SummaryOutputSchema.safeParse({
+ title: "Summary",
+ narrative: "Too short",
+ keyDecisions: [],
+ filesModified: [],
+ concepts: [],
+ });
+ expect(result.success).toBe(false);
+ });
+ });
+
+ describe("SearchInputSchema", () => {
+ it("accepts valid search", () => {
+ expect(SearchInputSchema.safeParse({ query: "auth" }).success).toBe(true);
+ });
+
+ it("accepts search with limit", () => {
+ expect(
+ SearchInputSchema.safeParse({ query: "auth", limit: 10 }).success,
+ ).toBe(true);
+ });
+
+ it("rejects empty query", () => {
+ expect(SearchInputSchema.safeParse({ query: "" }).success).toBe(false);
+ });
+ });
+
+ describe("ContextInputSchema", () => {
+ it("accepts valid input", () => {
+ expect(
+ ContextInputSchema.safeParse({
+ sessionId: "ses_1",
+ project: "proj",
+ }).success,
+ ).toBe(true);
+ });
+ });
+
+ describe("RememberInputSchema", () => {
+ it("accepts valid input", () => {
+ expect(
+ RememberInputSchema.safeParse({
+ content: "Always use TypeScript",
+ type: "preference",
+ }).success,
+ ).toBe(true);
+ });
+
+ it("rejects empty content", () => {
+ expect(
+ RememberInputSchema.safeParse({ content: "" }).success,
+ ).toBe(false);
+ });
+ });
+});
+
+describe("Validator", () => {
+ it("returns valid with correct data", () => {
+ const result = validateInput(SearchInputSchema, { query: "test" }, "search");
+ expect(result.valid).toBe(true);
+ if (result.valid) {
+ expect(result.data.query).toBe("test");
+ }
+ });
+
+ it("returns invalid with error details", () => {
+ const result = validateInput(SearchInputSchema, { query: "" }, "search");
+ expect(result.valid).toBe(false);
+ if (!result.valid) {
+ expect(result.result.functionId).toBe("search");
+ expect(result.result.errors.length).toBeGreaterThan(0);
+ }
+ });
+
+ it("validateOutput works same as validateInput", () => {
+ const result = validateOutput(
+ CompressOutputSchema,
+ {
+ type: "file_edit",
+ title: "Test",
+ facts: ["a"],
+ narrative: "A long enough narrative",
+ concepts: [],
+ files: [],
+ importance: 5,
+ },
+ "compress",
+ );
+ expect(result.valid).toBe(true);
+ });
+});
+
+describe("Quality Scoring", () => {
+ describe("scoreCompression", () => {
+ it("returns 0 for empty object", () => {
+ expect(scoreCompression({})).toBe(0);
+ });
+
+ it("returns 100 for perfect observation", () => {
+ const score = scoreCompression({
+ type: "file_edit",
+ title: "A good title",
+ facts: ["fact 1", "fact 2", "fact 3"],
+ narrative: "A narrative that is definitely more than fifty characters long and provides good context",
+ concepts: ["auth", "jwt"],
+ importance: 7,
+ });
+ expect(score).toBe(100);
+ });
+
+ it("scores partial observations between 0 and 100", () => {
+ const score = scoreCompression({
+ title: "Test",
+ facts: ["one"],
+ narrative: "Short but valid narrative",
+ });
+ expect(score).toBeGreaterThan(0);
+ expect(score).toBeLessThan(100);
+ });
+ });
+
+ describe("scoreSummary", () => {
+ it("returns 0 for empty object", () => {
+ expect(scoreSummary({})).toBe(0);
+ });
+
+ it("returns high score for complete summary", () => {
+ const score = scoreSummary({
+ title: "Session Summary Title",
+ narrative:
+ "This is a detailed narrative about what happened during the session with enough content to be meaningful and complete for review purposes",
+ keyDecisions: ["Used JWT for auth", "Chose PostgreSQL"],
+ filesModified: ["src/auth.ts", "src/db.ts"],
+ concepts: ["authentication", "database"],
+ });
+ expect(score).toBeGreaterThanOrEqual(90);
+ });
+ });
+
+ describe("scoreContextRelevance", () => {
+ it("returns 0 for empty context", () => {
+ expect(scoreContextRelevance("", "proj")).toBe(0);
+ });
+
+ it("scores higher when project is mentioned", () => {
+ const withProject = scoreContextRelevance(
+ "<context>This is for my-project with details</context>",
+ "my-project",
+ );
+ const without = scoreContextRelevance(
+ "<context>Some generic context details</context>",
+ "my-project",
+ );
+ expect(withProject).toBeGreaterThan(without);
+ });
+
+ it("scores higher with more XML sections", () => {
+ const multi = scoreContextRelevance(
+ "<summary>A</summary><observations>B</observations><memories>C</memories><patterns>D</patterns>",
+ "test",
+ );
+ const single = scoreContextRelevance("<summary>A</summary>", "test");
+ expect(multi).toBeGreaterThan(single);
+ });
+ });
+});