Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

feat(diag): smart-search followup-rate proxy + reader-failure counter (#786)

Rohit Ghumare9a4bdbe0e802
diff --git a/AGENTS.md b/AGENTS.md
index 0330b14f..ae0d577f 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -117,7 +117,7 @@ Hook scripts in `src/hooks/` are standalone Node.js scripts (no iii-sdk import).
## Current Stats (v0.9.16)
- 53 MCP tools (8 visible by default, `AGENTMEMORY_TOOLS=all` for all)
-- 125 REST endpoints
+- 126 REST endpoints
- 6 MCP resources, 3 MCP prompts
- 12 hooks, 4 skills
- 50+ iii functions
diff --git a/README.md b/README.md
index 862e540f..d1eaa35f 100644
--- a/README.md
+++ b/README.md
@@ -1429,7 +1429,7 @@ Create `~/.agentmemory/.env`:
<h2 id="api"><picture><source media="(prefers-color-scheme: dark)" srcset="assets/tags/light/section-api.svg"><img src="assets/tags/section-api.svg" alt="API" height="32" /></picture></h2>
-125 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer <secret>` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers.
+126 endpoints on port `3111`. The REST API binds to `127.0.0.1` by default. Protected endpoints require `Authorization: Bearer <secret>` when `AGENTMEMORY_SECRET` is set, and mesh sync endpoints require `AGENTMEMORY_SECRET` on both peers.
<details>
<summary>Key endpoints</summary>
diff --git a/src/cli.ts b/src/cli.ts
index 98244be0..62d39734 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -162,6 +162,9 @@ Environment:
AGENTMEMORY_USE_DOCKER=1 Prefer the bundled docker-compose path over the
native iii-engine binary on first run.
AGENTMEMORY_III_VERSION Override pinned iii-engine version (default ${IIPINNED_VERSION}).
+ AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS
+ Window (seconds) for the smart-search follow-up diagnostic
+ (default 30). Long values overcount, short values undercount.
Quick start:
npx @agentmemory/agentmemory # start with local iii-engine or Docker
@@ -1271,12 +1274,13 @@ async function runStatus() {
}
try {
- const [healthRes, sessionsRes, graphRes, memoriesRes, flagsRes] = await Promise.all([
+ const [healthRes, sessionsRes, graphRes, memoriesRes, flagsRes, followupRes] = await Promise.all([
apiFetch<any>(base, "health"),
apiFetch<any>(base, "sessions"),
apiFetch<any>(base, "graph/stats"),
apiFetch<any>(base, "memories?count=true"),
apiFetch<any>(base, "config/flags"),
+ apiFetch<any>(base, "diagnostics/followup"),
]);
if (typeof healthRes?.viewerPort === "number") {
@@ -1337,6 +1341,16 @@ async function runStatus() {
flagRows.forEach((r: string) => lines.push(r));
}
+ if (followupRes && Number.isFinite(followupRes.agentInitiatedSearches)) {
+ const total = Number(followupRes.agentInitiatedSearches) || 0;
+ const hits = Number(followupRes.followupWithinWindow) || 0;
+ const pct = total > 0 ? Math.round((hits / total) * 100) : 0;
+ lines.push("");
+ lines.push(
+ `Followup rate: ${hits}/${total} (${pct}%) within ${followupRes.windowSeconds}s — directional, may overcount on refinement`,
+ );
+ }
+
p.note(lines.join("\n"), "agentmemory");
} catch (err) {
p.log.error(err instanceof Error ? err.message : String(err));
diff --git a/src/config.ts b/src/config.ts
index 1fe70465..b3f8882d 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -322,6 +322,20 @@ export function getGraphBatchSize(): number {
return safeParseInt(getMergedEnv()["GRAPH_EXTRACTION_BATCH_SIZE"], 10);
}
+// #771: window for the smart-search followup-rate diagnostic. A second
+// search arriving within this many seconds (with disjoint results)
+// counts as a "follow-up" — a directional signal that the first result
+// set didn't satisfy. Long values overcount (legitimate refinement
+// looks like a follow-up); short values undercount.
+const FOLLOWUP_WINDOW_DEFAULT_SECONDS = 30;
+
+export function getFollowupWindowSeconds(): number {
+ return safeParseInt(
+ getMergedEnv()["AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS"],
+ FOLLOWUP_WINDOW_DEFAULT_SECONDS,
+ );
+}
+
export function isConsolidationEnabled(): boolean {
const env = getMergedEnv();
const explicit = env["CONSOLIDATION_ENABLED"];
diff --git a/src/functions/recent-searches-sweep.ts b/src/functions/recent-searches-sweep.ts
new file mode 100644
index 00000000..03df192c
--- /dev/null
+++ b/src/functions/recent-searches-sweep.ts
@@ -0,0 +1,72 @@
+import type { ISdk } from "iii-sdk";
+import type { StateKV } from "../state/kv.js";
+import { KV } from "../state/schema.js";
+import { logger } from "../logger.js";
+import { getFollowupStats, type RecentSearch } from "./smart-search.js";
+import { getFollowupWindowSeconds } from "../config.js";
+
+// #771: TTL sweep for the followup-rate diagnostic scope. `recentSearches`
+// only needs the most recent entry per session, but stale rows accumulate
+// when sessions go idle. Hourly sweep deletes rows whose last update is
+// older than the retention window.
+const RETENTION_MS = 24 * 60 * 60 * 1000;
+
+export function registerRecentSearchesSweepFunction(
+ sdk: ISdk,
+ kv: StateKV,
+): void {
+ sdk.registerFunction(
+ "mem::diagnostic::recent-searches-sweep",
+ async (): Promise<{ success: true; swept: number; skipped: number }> => {
+ const cutoff = Date.now() - RETENTION_MS;
+ const rows = await kv
+ .list<Partial<RecentSearch>>(KV.recentSearches)
+ .catch(() => []);
+ let swept = 0;
+ let skipped = 0;
+ for (const row of rows) {
+ if (!row || typeof row.sessionId !== "string" || !row.sessionId) {
+ skipped++;
+ continue;
+ }
+ const at = typeof row.at === "number" ? row.at : 0;
+ if (at >= cutoff) continue;
+ try {
+ await kv.delete(KV.recentSearches, row.sessionId);
+ swept++;
+ } catch (err) {
+ logger.warn("recent-searches sweep delete failed", {
+ sessionId: row.sessionId,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ }
+ }
+ if (swept > 0 || skipped > 0) {
+ logger.info("Recent-searches sweep complete", { swept, skipped });
+ }
+ return { success: true, swept, skipped };
+ },
+ );
+
+ // #771: read-back surface for `agentmemory status` and external
+ // dashboards that don't go through the OTEL collector.
+ sdk.registerFunction(
+ "mem::diagnostic::followup-stats",
+ async (): Promise<{
+ success: true;
+ windowSeconds: number;
+ agentInitiatedSearches: number;
+ followupWithinWindow: number;
+ rate: number;
+ }> => {
+ const stats = getFollowupStats();
+ return {
+ success: true,
+ windowSeconds: getFollowupWindowSeconds(),
+ agentInitiatedSearches: stats.agentInitiatedSearches,
+ followupWithinWindow: stats.followupWithinWindow,
+ rate: stats.rate,
+ };
+ },
+ );
+}
diff --git a/src/functions/smart-search.ts b/src/functions/smart-search.ts
index fbdaad0d..10a0a77c 100644
--- a/src/functions/smart-search.ts
+++ b/src/functions/smart-search.ts
@@ -8,9 +8,65 @@ import type {
} from "../types.js";
import { KV } from "../state/schema.js";
import { StateKV } from "../state/kv.js";
+import { withKeyedLock } from "../state/keyed-mutex.js";
import { recordAccessBatch } from "./access-tracker.js";
-import { getAgentId, isAgentScopeIsolated } from "../config.js";
+import {
+ getAgentId,
+ isAgentScopeIsolated,
+ getFollowupWindowSeconds,
+} from "../config.js";
import { logger } from "../logger.js";
+import { getCounters } from "../telemetry/setup.js";
+
+// #771: smart-search followup-rate diagnostic. Stored per session as
+// the most recent search payload, used to detect whether the next
+// search inside the window had a disjoint result set. sessionId is
+// duplicated into the row so the hourly sweep can delete by it
+// (StateKV.list returns values only).
+export interface RecentSearch {
+ sessionId: string;
+ query: string;
+ resultIds: string[];
+ at: number;
+}
+
+// Module-scope counter mirror so `mem::diagnostic::followup-stats` can
+// read the rate back without going through the OTEL collector. The
+// OTEL counter is still the canonical export; this is an in-process
+// convenience for `agentmemory status` + tests.
+const followupStats = {
+ followupWithinWindow: 0,
+ agentInitiatedSearches: 0,
+};
+
+// Tracks the in-flight detection promises so tests (and shutdown
+// flushes) can wait for all queued lock bodies to drain. The Set adds
+// when a detection is queued and removes when it settles; size === 0
+// means no pending detections.
+const pendingFollowups = new Set<Promise<void>>();
+
+export function getFollowupStats(): {
+ followupWithinWindow: number;
+ agentInitiatedSearches: number;
+ rate: number;
+} {
+ const total = followupStats.agentInitiatedSearches;
+ return {
+ ...followupStats,
+ rate: total > 0 ? followupStats.followupWithinWindow / total : 0,
+ };
+}
+
+export async function flushPendingFollowups(): Promise<void> {
+ // Snapshot the current pending set; new detections queued after the
+ // snapshot run in a fresh batch.
+ await Promise.all(Array.from(pendingFollowups));
+}
+
+export function resetFollowupStatsForTests(): void {
+ followupStats.followupWithinWindow = 0;
+ followupStats.agentInitiatedSearches = 0;
+}
// Compact mode trims each lesson's content for at-a-glance display. The
// full content is fetched via memory_lesson_recall when the caller needs it.
@@ -32,6 +88,13 @@ export function registerSmartSearchFunction(
// roles through one server. "*" opts out of the env-default
// scope and returns hits from every agent.
agentId?: string;
+ // #771: session anchor for the followup-rate diagnostic. The
+ // API trigger fills this from req.body / headers; direct
+ // sdk.trigger callers can pass it explicitly.
+ sessionId?: string;
+ // #771: marks viewer-originated searches so the diagnostic
+ // ignores them — only agent-initiated re-queries should count.
+ source?: string;
}) => {
// Compute the agent filter once, up front. Both the expandIds
@@ -141,6 +204,54 @@ export function registerSmartSearchFunction(
compact.map((r) => r.obsId),
);
+ // #771: followup-rate diagnostic. Only fires for agent-initiated
+ // searches that carry a sessionId — viewer-originated searches
+ // (source === "viewer") and direct-sdk callers without a session
+ // anchor are skipped. The result-set comparison uses obsIds: a
+ // disjoint set under the window suggests the previous call's
+ // results were not used, which is our directional proxy for
+ // reader-failure-with-evidence.
+ if (
+ data.sessionId &&
+ typeof data.sessionId === "string" &&
+ data.source !== "viewer" &&
+ compact.length > 0
+ ) {
+ // Skip detection when retrieval returned nothing: an empty
+ // result set is a retrieval failure, not a reader-failure
+ // signal. Counting it as "disjoint from prior" would inflate
+ // the rate every time search returns no hits.
+ followupStats.agentInitiatedSearches++;
+ // Off the critical response path. The withKeyedLock(sessionId)
+ // call serializes detection per session, so two rapid
+ // back-to-back searches from the same agent still see ordered
+ // prior-row writes — the second call's lock body queues
+ // behind the first's. Other sessions run in parallel.
+ const sessionIdForFollowup = data.sessionId;
+ const queryForFollowup = data.query;
+ const compactForFollowup = compact;
+ const detection = withKeyedLock(
+ `recent-searches:${sessionIdForFollowup}`,
+ () =>
+ detectFollowup(
+ kv,
+ sessionIdForFollowup,
+ queryForFollowup,
+ compactForFollowup,
+ ),
+ )
+ .catch((err) => {
+ logger.warn("Smart search followup detection failed", {
+ sessionId: sessionIdForFollowup,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ })
+ .finally(() => {
+ pendingFollowups.delete(detection);
+ });
+ pendingFollowups.add(detection);
+ }
+
logger.info("Smart search compact", {
query: data.query,
results: compact.length,
@@ -189,6 +300,46 @@ async function recallLessons(
}
}
+async function detectFollowup(
+ kv: StateKV,
+ sessionId: string,
+ query: string,
+ compact: CompactSearchResult[],
+): Promise<void> {
+ const now = Date.now();
+ const windowMs = Math.max(1, getFollowupWindowSeconds()) * 1000;
+ const currentIds = compact.map((r) => r.obsId);
+ const current: RecentSearch = { sessionId, query, resultIds: currentIds, at: now };
+
+ const prior = await kv
+ .get<RecentSearch>(KV.recentSearches, sessionId)
+ .catch(() => null);
+
+ await kv.set(KV.recentSearches, sessionId, current);
+
+ if (!prior || typeof prior.at !== "number") return;
+ if (now - prior.at > windowMs) return;
+ // Same query inside the window is a retry, not a follow-up; skip so a
+ // duplicate request from a flaky client doesn't inflate the metric.
+ if (typeof prior.query === "string" && prior.query === query) return;
+
+ const priorIds = Array.isArray(prior.resultIds) ? prior.resultIds : [];
+ const priorSet = new Set(priorIds);
+ const hasOverlap = currentIds.some((id) => priorSet.has(id));
+ if (hasOverlap) return;
+
+ getCounters().smartSearchFollowupWithinWindow.add(1);
+ followupStats.followupWithinWindow++;
+ logger.info("Smart search followup detected", {
+ sessionId,
+ windowSeconds: Math.round(windowMs / 1000),
+ priorQuery: prior.query,
+ nextQuery: query,
+ priorResultCount: priorIds.length,
+ nextResultCount: currentIds.length,
+ });
+}
+
async function findObservation(
kv: StateKV,
obsId: string,
diff --git a/src/index.ts b/src/index.ts
index d20d4693..cef06a24 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -50,6 +50,7 @@ import { registerEvictFunction } from "./functions/evict.js";
import { registerRelationsFunction } from "./functions/relations.js";
import { registerTimelineFunction } from "./functions/timeline.js";
import { registerSmartSearchFunction } from "./functions/smart-search.js";
+import { registerRecentSearchesSweepFunction } from "./functions/recent-searches-sweep.js";
import { registerProfileFunction } from "./functions/profile.js";
import { registerAutoForgetFunction } from "./functions/auto-forget.js";
import { registerExportImportFunction } from "./functions/export-import.js";
@@ -366,6 +367,7 @@ async function main() {
registerSmartSearchFunction(sdk, kv, (query, limit) =>
hybridSearch.search(query, limit),
);
+ registerRecentSearchesSweepFunction(sdk, kv);
registerApiTriggers(sdk, kv, secret, metricsStore, provider);
registerEventTriggers(sdk, kv);
@@ -516,7 +518,7 @@ async function main() {
`Ready. ${embeddingProvider ? "Triple-stream (BM25+Vector+Graph)" : "BM25+Graph"} search active.`,
);
bootLog(
- `REST API: 125 endpoints at http://localhost:${config.restPort}/agentmemory/*`,
+ `REST API: 126 endpoints at http://localhost:${config.restPort}/agentmemory/*`,
);
bootLog(
`MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`,
@@ -563,6 +565,20 @@ async function main() {
insightDecayTimer.unref();
}
+ // #771: hourly TTL sweep for the followup-rate diagnostic. The
+ // recent-searches scope only needs the last entry per session;
+ // sweeping anything older than the retention window keeps the scope
+ // from growing unbounded across long-lived deployments.
+ const recentSearchesSweepTimer = setInterval(async () => {
+ try {
+ await sdk.trigger({
+ function_id: "mem::diagnostic::recent-searches-sweep",
+ payload: {},
+ });
+ } catch {}
+ }, 60 * 60 * 1000);
+ recentSearchesSweepTimer.unref();
+
if (isConsolidationEnabled()) {
const consolidationTimer = setInterval(async () => {
try {
diff --git a/src/state/schema.ts b/src/state/schema.ts
index 8d0d90f0..2000d52c 100644
--- a/src/state/schema.ts
+++ b/src/state/schema.ts
@@ -47,6 +47,9 @@ export const KV = {
globalSlots: "mem:slots:global",
state: "mem:state",
commits: "mem:commits",
+ // #771: tracks the most recent smart-search call per session, used by
+ // the followup-rate diagnostic. Key = sessionId. TTL-swept hourly.
+ recentSearches: "mem:recent-searches",
} as const;
export const STREAM = {
diff --git a/src/telemetry/setup.ts b/src/telemetry/setup.ts
index 0785b614..fc55b4f5 100644
--- a/src/telemetry/setup.ts
+++ b/src/telemetry/setup.ts
@@ -39,6 +39,19 @@ interface Counters {
auditLog: Counter;
snapshotCreate: Counter;
governanceDelete: Counter;
+ // #771: smart-search follow-up proxy. Incremented when a session
+ // issues a second smart-search inside the configured window AND the
+ // new result set has zero overlap with the previous one (a
+ // directional signal for "the first results didn't satisfy"). Treat
+ // as directional, not absolute — legitimate query refinement counts
+ // here too.
+ smartSearchFollowupWithinWindow: Counter;
+ // #771: benchmark-mode counter. Incremented by the benchmark scorer
+ // when judge_correct === false AND the gold-evidence-IDs are a
+ // subset of the retrieved-context-IDs (reader missed the answer
+ // despite retrieval being correct). Never incremented by core in
+ // live use; reserved here so dashboards keep a stable name.
+ readerFailureWithEvidence: Counter;
}
interface Histograms {
@@ -81,6 +94,8 @@ const COUNTER_NAMES: Array<[keyof Counters, string]> = [
["auditLog", "audit.log"],
["snapshotCreate", "snapshot.create"],
["governanceDelete", "governance.delete"],
+ ["smartSearchFollowupWithinWindow", "smart_search.followup_within_window_total"],
+ ["readerFailureWithEvidence", "reader_failure_with_evidence_total"],
];
const HISTOGRAM_NAMES: Array<[keyof Histograms, string]> = [
@@ -92,6 +107,24 @@ const HISTOGRAM_NAMES: Array<[keyof Histograms, string]> = [
["vectorSearchLatency", "vector_search.latency_ms"],
];
+// Accessors so functions outside `initMetrics`'s closure can record into
+// the same counter / histogram instances after init. Before init both
+// return no-op fallbacks so call sites stay safe in tests and during
+// the boot window.
+export function getCounters(): Counters {
+ if (counters) return counters;
+ return Object.fromEntries(
+ COUNTER_NAMES.map(([key]) => [key, NOOP_COUNTER]),
+ ) as unknown as Counters;
+}
+
+export function getHistograms(): Histograms {
+ if (histograms) return histograms;
+ return Object.fromEntries(
+ HISTOGRAM_NAMES.map(([key]) => [key, NOOP_HISTOGRAM]),
+ ) as unknown as Histograms;
+}
+
export function initMetrics(getMeter?: (name: string) => Meter): {
counters: Counters;
histograms: Histograms;
diff --git a/src/triggers/api.ts b/src/triggers/api.ts
index ea817164..c8643558 100644
--- a/src/triggers/api.ts
+++ b/src/triggers/api.ts
@@ -1096,9 +1096,18 @@ export function registerApiTriggers(
config: { api_path: "/agentmemory/evict", http_method: "POST" },
});
- sdk.registerFunction("api::smart-search",
+ sdk.registerFunction("api::smart-search",
async (
- req: ApiRequest<{ query?: string; expandIds?: string[]; limit?: number }>,
+ req: ApiRequest<{
+ query?: string;
+ expandIds?: Array<string | { obsId: string; sessionId: string }>;
+ limit?: number;
+ project?: string;
+ includeLessons?: boolean;
+ agentId?: string;
+ sessionId?: string;
+ source?: string;
+ }>,
): Promise<Response> => {
const authErr = checkAuth(req, secret);
if (authErr) return authErr;
@@ -1111,7 +1120,27 @@ export function registerApiTriggers(
body: { error: "query or expandIds is required" },
};
}
- const result = await sdk.trigger({ function_id: "mem::smart-search", payload: req.body });
+ // #771: route the X-Agentmemory-Source header into the payload so
+ // the followup-rate diagnostic can skip viewer-originated calls.
+ // Body wins if both are set (advanced callers explicitly override).
+ const headers = (req.headers || {}) as Record<string, string | string[] | undefined>;
+ const sourceHeader = headers["x-agentmemory-source"] ?? headers["X-Agentmemory-Source"];
+ const sourceFromHeader = Array.isArray(sourceHeader) ? sourceHeader[0] : sourceHeader;
+ // Whitelist payload fields explicitly — REST endpoints never pass
+ // the raw request body through to sdk.trigger (AGENTS.md security
+ // section). Drops unknown fields so a misbehaving client can't
+ // inject downstream-only options.
+ const payload = {
+ query: req.body?.query,
+ expandIds: req.body?.expandIds,
+ limit: req.body?.limit,
+ project: req.body?.project,
+ includeLessons: req.body?.includeLessons,
+ agentId: req.body?.agentId,
+ sessionId: req.body?.sessionId,
+ source: req.body?.source ?? sourceFromHeader,
+ };
+ const result = await sdk.trigger({ function_id: "mem::smart-search", payload });
return { status_code: 200, body: result };
},
);
@@ -1121,6 +1150,38 @@ export function registerApiTriggers(
config: { api_path: "/agentmemory/smart-search", http_method: "POST" },
});
+ // #771: read-back endpoint for the followup-rate diagnostic. Returns
+ // a directional signal — overcounts on legitimate query refinement —
+ // so help text + the CLI status line carry the same caveat.
+ sdk.registerFunction("api::diagnostic-followup",
+ async (req: ApiRequest): Promise<Response> => {
+ const authErr = checkAuth(req, secret);
+ if (authErr) return authErr;
+ const result = await sdk.trigger({
+ function_id: "mem::diagnostic::followup-stats",
+ payload: {},
+ });
+ return {
+ status_code: 200,
+ body: {
+ ...(result as Record<string, unknown>),
+ caveat:
+ "Directional signal: overcounts on legitimate query refinement. " +
+ "Tune via AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS.",
+ },
+ };
+ },
+ );
+ sdk.registerTrigger({
+ type: "http",
+ function_id: "api::diagnostic-followup",
+ config: {
+ api_path: "/agentmemory/diagnostics/followup",
+ http_method: "GET",
+ middleware_function_ids: ["middleware::api-auth"],
+ },
+ });
+
sdk.registerFunction("api::timeline",
async (
req: ApiRequest<{
diff --git a/test/diagnostic-followup-rate.test.ts b/test/diagnostic-followup-rate.test.ts
new file mode 100644
index 00000000..d747e330
--- /dev/null
+++ b/test/diagnostic-followup-rate.test.ts
@@ -0,0 +1,300 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+
+vi.mock("../src/logger.js", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+import {
+ registerSmartSearchFunction,
+ getFollowupStats,
+ resetFollowupStatsForTests,
+ flushPendingFollowups,
+} from "../src/functions/smart-search.js";
+import { registerRecentSearchesSweepFunction } from "../src/functions/recent-searches-sweep.js";
+import { KV } from "../src/state/schema.js";
+import type { HybridSearchResult } from "../src/types.js";
+
+function mockKV() {
+ const store = new Map<string, Map<string, unknown>>();
+ return {
+ get: async <T>(scope: string, key: string): Promise<T | null> =>
+ (store.get(scope)?.get(key) as T) ?? null,
+ set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+ if (!store.has(scope)) store.set(scope, new Map());
+ store.get(scope)!.set(key, data);
+ return data;
+ },
+ delete: async (scope: string, key: string): Promise<void> => {
+ store.get(scope)?.delete(key);
+ },
+ list: async <T>(scope: string): Promise<T[]> =>
+ Array.from(store.get(scope)?.values() ?? []) as T[],
+ getStore: () => store,
+ };
+}
+
+function mockSdk(kv: ReturnType<typeof mockKV>) {
+ const functions = new Map<string, Function>();
+ const sdk = {
+ registerFunction: (
+ idOrOpts: string | { id: string },
+ handler: Function,
+ ) => {
+ const id = typeof idOrOpts === "string" ? idOrOpts : idOrOpts.id;
+ functions.set(id, handler);
+ },
+ registerTrigger: () => {},
+ trigger: async (
+ idOrInput: string | { function_id: string; payload?: unknown },
+ data?: unknown,
+ ) => {
+ const id = typeof idOrInput === "string" ? idOrInput : idOrInput.function_id;
+ const payload =
+ typeof idOrInput === "string" ? data : (idOrInput as any).payload;
+ const fn = functions.get(id);
+ if (!fn) {
+ if (id === "mem::lesson-recall") return { success: true, lessons: [] };
+ throw new Error(`No function: ${id}`);
+ }
+ const result = await fn(payload);
+ // smart-search now runs followup detection off the critical
+ // response path; drain it before returning so test assertions
+ // see consistent state.
+ if (id === "mem::smart-search") await flushPendingFollowups();
+ return result;
+ },
+ } as any;
+ void kv;
+ return sdk;
+}
+
+function makeHit(obsId: string, sessionId = "ses_1"): HybridSearchResult {
+ return {
+ observation: {
+ id: obsId,
+ sessionId,
+ timestamp: new Date().toISOString(),
+ title: `obs ${obsId}`,
+ narrative: "n",
+ type: "pattern",
+ concepts: [],
+ files: [],
+ } as any,
+ sessionId,
+ combinedScore: 0.8,
+ } as HybridSearchResult;
+}
+
+describe("Smart-search followup-rate diagnostic (#771)", () => {
+ let sdk: any;
+ let kv: ReturnType<typeof mockKV>;
+ let searchResults: HybridSearchResult[];
+
+ beforeEach(() => {
+ delete process.env.AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS;
+ resetFollowupStatsForTests();
+ kv = mockKV();
+ sdk = mockSdk(kv);
+ searchResults = [];
+ registerSmartSearchFunction(sdk, kv as any, async () => searchResults);
+ registerRecentSearchesSweepFunction(sdk, kv as any);
+ });
+
+ afterEach(() => {
+ delete process.env.AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS;
+ });
+
+ it("records the first agent-initiated search but does not flag it as a followup", async () => {
+ searchResults = [makeHit("obs_a"), makeHit("obs_b")];
+ await sdk.trigger("mem::smart-search", {
+ query: "auth flow",
+ sessionId: "ses_1",
+ });
+
+ const stats = getFollowupStats();
+ expect(stats.agentInitiatedSearches).toBe(1);
+ expect(stats.followupWithinWindow).toBe(0);
+
+ const stored = await kv.get(KV.recentSearches, "ses_1");
+ expect(stored).not.toBeNull();
+ expect((stored as any).sessionId).toBe("ses_1");
+ expect((stored as any).query).toBe("auth flow");
+ });
+
+ it("flags a follow-up when the second search inside the window returns a disjoint set", async () => {
+ searchResults = [makeHit("obs_a"), makeHit("obs_b")];
+ await sdk.trigger("mem::smart-search", {
+ query: "auth flow",
+ sessionId: "ses_1",
+ });
+
+ searchResults = [makeHit("obs_c"), makeHit("obs_d")];
+ await sdk.trigger("mem::smart-search", {
+ query: "token expiry handling",
+ sessionId: "ses_1",
+ });
+
+ const stats = getFollowupStats();
+ expect(stats.agentInitiatedSearches).toBe(2);
+ expect(stats.followupWithinWindow).toBe(1);
+ expect(stats.rate).toBeCloseTo(0.5);
+ });
+
+ it("does not flag a follow-up when result sets overlap", async () => {
+ searchResults = [makeHit("obs_a"), makeHit("obs_b")];
+ await sdk.trigger("mem::smart-search", {
+ query: "auth flow",
+ sessionId: "ses_1",
+ });
+
+ searchResults = [makeHit("obs_b"), makeHit("obs_c")];
+ await sdk.trigger("mem::smart-search", {
+ query: "auth token",
+ sessionId: "ses_1",
+ });
+
+ const stats = getFollowupStats();
+ expect(stats.followupWithinWindow).toBe(0);
+ });
+
+ it("does not flag a follow-up on an identical re-query (retry, not follow-up)", async () => {
+ searchResults = [makeHit("obs_a")];
+ await sdk.trigger("mem::smart-search", {
+ query: "auth flow",
+ sessionId: "ses_1",
+ });
+
+ // Different result set on the retry (e.g. flaky search index), but
+ // same query — still not a follow-up.
+ searchResults = [makeHit("obs_b")];
+ await sdk.trigger("mem::smart-search", {
+ query: "auth flow",
+ sessionId: "ses_1",
+ });
+
+ expect(getFollowupStats().followupWithinWindow).toBe(0);
+ });
+
+ it("does not flag a follow-up when prior search is outside the window", async () => {
+ process.env.AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS = "1";
+ searchResults = [makeHit("obs_a")];
+ await sdk.trigger("mem::smart-search", {
+ query: "first",
+ sessionId: "ses_1",
+ });
+
+ // Backdate the stored search so the window has elapsed.
+ const stored = (await kv.get(KV.recentSearches, "ses_1")) as any;
+ stored.at = Date.now() - 5_000;
+ await kv.set(KV.recentSearches, "ses_1", stored);
+
+ searchResults = [makeHit("obs_b")];
+ await sdk.trigger("mem::smart-search", {
+ query: "second",
+ sessionId: "ses_1",
+ });
+
+ expect(getFollowupStats().followupWithinWindow).toBe(0);
+ });
+
+ it("skips viewer-originated searches (source === 'viewer')", async () => {
+ searchResults = [makeHit("obs_a")];
+ await sdk.trigger("mem::smart-search", {
+ query: "from viewer",
+ sessionId: "ses_1",
+ source: "viewer",
+ });
+
+ expect(getFollowupStats().agentInitiatedSearches).toBe(0);
+ // Viewer call shouldn't write to recent-searches either, otherwise
+ // a subsequent agent call would treat the viewer search as prior.
+ expect(await kv.get(KV.recentSearches, "ses_1")).toBeNull();
+ });
+
+ it("skips searches without a sessionId (direct sdk callers)", async () => {
+ searchResults = [makeHit("obs_a")];
+ await sdk.trigger("mem::smart-search", { query: "no session" });
+
+ expect(getFollowupStats().agentInitiatedSearches).toBe(0);
+ });
+
+ it("recent-searches sweep deletes rows older than 24h, keeps fresh ones", async () => {
+ const fresh = {
+ sessionId: "ses_fresh",
+ query: "x",
+ resultIds: [],
+ at: Date.now() - 1_000,
+ };
+ const stale = {
+ sessionId: "ses_stale",
+ query: "x",
+ resultIds: [],
+ at: Date.now() - 25 * 60 * 60 * 1000,
+ };
+ await kv.set(KV.recentSearches, fresh.sessionId, fresh);
+ await kv.set(KV.recentSearches, stale.sessionId, stale);
+
+ const result = (await sdk.trigger(
+ "mem::diagnostic::recent-searches-sweep",
+ {},
+ )) as { swept: number };
+
+ expect(result.swept).toBe(1);
+ expect(await kv.get(KV.recentSearches, "ses_fresh")).not.toBeNull();
+ expect(await kv.get(KV.recentSearches, "ses_stale")).toBeNull();
+ });
+
+ it("skips detection when current results are empty (retrieval failure, not reader failure)", async () => {
+ searchResults = [makeHit("obs_a"), makeHit("obs_b")];
+ await sdk.trigger("mem::smart-search", {
+ query: "first",
+ sessionId: "ses_1",
+ });
+
+ // Empty result set on the next call. Without the empty-skip guard
+ // the empty-vs-prior comparison would be vacuously "disjoint" and
+ // inflate the rate. Skip detection entirely.
+ searchResults = [];
+ await sdk.trigger("mem::smart-search", {
+ query: "second",
+ sessionId: "ses_1",
+ });
+
+ const stats = getFollowupStats();
+ // Only the first call counts as agent-initiated; the empty-result
+ // second call is skipped entirely.
+ expect(stats.agentInitiatedSearches).toBe(1);
+ expect(stats.followupWithinWindow).toBe(0);
+ });
+
+ it("followup-stats function returns the configured window and live counts", async () => {
+ process.env.AGENTMEMORY_FOLLOWUP_WINDOW_SECONDS = "45";
+ searchResults = [makeHit("obs_a")];
+ await sdk.trigger("mem::smart-search", {
+ query: "q1",
+ sessionId: "ses_1",
+ });
+ searchResults = [makeHit("obs_b")];
+ await sdk.trigger("mem::smart-search", {
+ query: "q2",
+ sessionId: "ses_1",
+ });
+
+ const stats = (await sdk.trigger(
+ "mem::diagnostic::followup-stats",
+ {},
+ )) as {
+ success: boolean;
+ windowSeconds: number;
+ agentInitiatedSearches: number;
+ followupWithinWindow: number;
+ rate: number;
+ };
+
+ expect(stats.success).toBe(true);
+ expect(stats.windowSeconds).toBe(45);
+ expect(stats.agentInitiatedSearches).toBe(2);
+ expect(stats.followupWithinWindow).toBe(1);
+ expect(stats.rate).toBeCloseTo(0.5);
+ });
+});