#1 Persistent memory for AI coding agents based on real-world benchmarks
TypeScript agentmemoryagentsaiclaudeclaudecodecodexcopilotcursorgenaiharnesshermesmemoryompopenclawopencodepi
import { pinGraphState } from "../state/graph-generation.js";
import { persistGraphRecords, validateGraphRecords } from "../state/graph-ingestion.js";
import { MAX_IMPORT_OBS_PER_SESSION } from "../config.js";
import type { IIIClient } from "iii-sdk";
import { markCaptureEventDeleted } from "../capture/event-record.js";
import type {
Session,
CompressedObservation,
Memory,
SessionSummary,
ProjectProfile,
ExportData,
Action,
ActionEdge,
Routine,
Signal,
Checkpoint,
Sentinel,
Sketch,
Crystal,
Facet,
Lesson,
Insight,
ExportPagination,
AccessLogExport,
} from "../types.js";
import { importOrigin } from "../types.js";
import { normalizeAccessLog } from "./access-tracker.js";
import { KV } from "../state/schema.js";
import { checkPayloadFrameSize, oversizedPayloadError, payloadByteLength, SAFE_PAYLOAD_BYTES } from "../state/frame-guard.js";
import { StateKV } from "../state/kv.js";
import {
addSessionToProjectIndex,
removeSessionFromProjectIndex,
} from "../state/session-index.js";
import {
indexObservationSession,
unindexObservationSession,
} from "../state/obs-index.js";
import { withKeyedLocks } from "../state/keyed-mutex.js";
import { VERSION } from "../version.js";
import { recordAudit } from "./audit.js";
import { indexRecords } from "./search.js";
import { resetLessonIndex } from "./lessons.js";
import { logger } from "../logger.js";
import { budgetImportedObservationSources } from "./observation-source-budget.js";
// Bounded-concurrency chunk size for the import delete/write loops. A
// "replace" or "merge" of a large export (up to MAX_TOTAL_OBSERVATIONS,
// ~500k) would otherwise issue hundreds of thousands of sequential state
// round-trips and blow the 180s function timeout, leaving partial state.
// 20 keeps per-chunk fan-out low enough not to overwhelm the state
// backend while collapsing wallclock by ~20x versus the serial path.
const IMPORT_CHUNK_SIZE = 20;
// Run `fn` over `items` in fixed-size chunks, awaiting each chunk before
// starting the next. Preserves ordering guarantees across chunks (chunk N
// fully settles before chunk N+1 begins) while parallelizing within a
// chunk. Settle sibling writes before propagating a failure so mutation
// locks are not released while state writes are still pending.
async function runChunked<T>(
items: readonly T[],
fn: (item: T) => Promise<void>,
): Promise<void> {
for (let i = 0; i < items.length; i += IMPORT_CHUNK_SIZE) {
const chunk = items.slice(i, i + IMPORT_CHUNK_SIZE);
const results = await Promise.allSettled(chunk.map(fn));
const failure = results.find((result) => result.status === "rejected");
if (failure?.status === "rejected") throw failure.reason;
}
}
const EXPORT_OVERSIZE_HINT =
"narrow the range with ?maxSessions / ?offset, page the rest with ?collectionLimit / ?collectionOffset, or ask for a subset with ?collections=";
const EXPORT_COLLECTIONS: ReadonlyArray<readonly [keyof ExportData, string]> = [
["graphNodes", KV.graphNodes],
["graphEdges", KV.graphEdges],
["semanticMemories", KV.semantic],
["proceduralMemories", KV.procedural],
["actions", KV.actions],
["actionEdges", KV.actionEdges],
["sentinels", KV.sentinels],
["sketches", KV.sketches],
["crystals", KV.crystals],
["facets", KV.facets],
["lessons", KV.lessons],
["insights", KV.insights],
["routines", KV.routines],
["signals", KV.signals],
["checkpoints", KV.checkpoints],
["accessLogs", KV.accessLog],
];
const EXPORT_COLLECTION_NAMES: ReadonlySet<string> = new Set([
"sessions",
"memories",
"summaries",
...EXPORT_COLLECTIONS.map(([field]) => field),
]);
function parseCollections(raw: unknown): ReadonlySet<string> | undefined {
if (raw === undefined || raw === null) return undefined;
const names = Array.isArray(raw) ? raw : String(raw).split(",");
return new Set(
names
.map((name) => String(name).trim())
.filter((name) => EXPORT_COLLECTION_NAMES.has(name)),
);
}
export class ExportBudget {
bytes = 0;
constructor(private readonly limit = SAFE_PAYLOAD_BYTES) {}
fits(items: readonly unknown[], key = ""): boolean {
this.bytes += key.length + 6;
for (const item of items) {
this.bytes += payloadByteLength(item) + 1;
if (this.bytes > this.limit) return false;
}
return this.bytes <= this.limit;
}
}
export function registerExportImportFunction(sdk: IIIClient, state: StateKV): void {
sdk.registerFunction("mem::export",
async (data?: {
maxSessions?: number;
offset?: number;
collectionLimit?: number;
collectionOffset?: number;
collections?: string[] | string;
}) => {
const kv = await pinGraphState(state);
const rawMax = Number(data?.maxSessions);
const maxSessions = Number.isFinite(rawMax) && rawMax > 0 ? Math.min(Math.floor(rawMax), 1000) : undefined;
const rawOffset = Number(data?.offset);
const offset = Number.isFinite(rawOffset) && rawOffset >= 0 ? Math.floor(rawOffset) : 0;
const rawCollectionLimit = Number(data?.collectionLimit);
const collectionLimit =
Number.isFinite(rawCollectionLimit) && rawCollectionLimit > 0
? Math.floor(rawCollectionLimit)
: undefined;
const rawCollectionOffset = Number(data?.collectionOffset);
const collectionOffset =
Number.isFinite(rawCollectionOffset) && rawCollectionOffset >= 0
? Math.floor(rawCollectionOffset)
: 0;
const selection = parseCollections(data?.collections);
const isSelected = (name: string): boolean =>
selection === undefined || selection.has(name);
const collectionTotals: Record<string, number> = {};
const sliceCollection = <T>(name: string, rows: T[]): T[] => {
collectionTotals[name] = rows.length;
if (!isSelected(name)) return [];
if (collectionLimit === undefined) return rows;
return rows.slice(collectionOffset, collectionOffset + collectionLimit);
};
const budget = new ExportBudget();
const refuse = (collection: string) => {
logger.warn("Export exceeds transport frame limit", {
bytesSoFar: budget.bytes,
collection,
});
return {
...oversizedPayloadError(budget.bytes, EXPORT_OVERSIZE_HINT),
stoppedAt: collection,
};
};
const optional = async <T>(scope: string): Promise<T[]> =>
kv.list<T>(scope).catch(() => [] as T[]);
const allSessions = await kv.list<Session>(KV.sessions);
const sessionWindow = maxSessions !== undefined
? allSessions.slice(offset, offset + maxSessions)
: allSessions;
const paginatedSessions = isSelected("sessions") ? sessionWindow : [];
if (!budget.fits(paginatedSessions)) return refuse("sessions");
const memories = sliceCollection("memories", await kv.list<Memory>(KV.memories));
if (!budget.fits(memories)) return refuse("memories");
const summaries = sliceCollection(
"summaries",
await kv.list<SessionSummary>(KV.summaries),
);
if (!budget.fits(summaries)) return refuse("summaries");
const observations: Record<string, CompressedObservation[]> = {};
for (const session of paginatedSessions) {
const obs = await optional<CompressedObservation>(
KV.observations(session.id),
);
if (obs.length === 0) continue;
if (!budget.fits(obs, session.id)) return refuse("observations");
observations[session.id] = obs;
}
const profiles: ProjectProfile[] = [];
const uniqueProjects = [...new Set(paginatedSessions.map((s) => s.project))];
const profileResults = await Promise.all(
uniqueProjects.map((project) =>
kv.get<ProjectProfile>(KV.profiles, project).catch(() => null),
),
);
for (const profile of profileResults) {
if (profile) profiles.push(profile);
}
if (!budget.fits(profiles)) return refuse("profiles");
const partial = collectionLimit !== undefined || selection !== undefined;
const exportedMemoryIds = new Set(memories.map((m) => m.id));
const accessLogsFor = (rows: AccessLogExport[]): AccessLogExport[] => {
collectionTotals["accessLogs"] = rows.length;
if (!isSelected("accessLogs")) return [];
return rows.filter((row) => exportedMemoryIds.has(row.memoryId));
};
const collections: Partial<ExportData> = {};
for (const [field, scope] of EXPORT_COLLECTIONS) {
const rows = await optional<unknown>(scope);
const items =
field === "accessLogs" && partial
? accessLogsFor(rows as AccessLogExport[])
: sliceCollection(field, rows);
if (items.length === 0) continue;
if (!budget.fits(items, field)) return refuse(field);
(collections as Record<string, unknown>)[field] = items;
}
const exportData: ExportData = {
version: VERSION,
exportedAt: new Date().toISOString(),
sessions: paginatedSessions,
observations,
memories,
summaries,
profiles: profiles.length > 0 ? profiles : undefined,
...collections,
};
if (maxSessions !== undefined) {
exportData.pagination = {
offset,
limit: maxSessions,
total: allSessions.length,
hasMore: offset + maxSessions < allSessions.length,
};
}
if (partial) {
exportData.collectionPagination = {
offset: collectionOffset,
...(collectionLimit !== undefined ? { limit: collectionLimit } : {}),
...(selection !== undefined ? { collections: [...selection] } : {}),
totals: collectionTotals,
hasMore:
collectionLimit !== undefined &&
Object.entries(collectionTotals).some(
([name, total]) =>
name !== "accessLogs" &&
isSelected(name) &&
collectionOffset + collectionLimit < total,
),
};
}
const oversized = checkPayloadFrameSize(exportData, EXPORT_OVERSIZE_HINT);
if (oversized) {
logger.warn("Export exceeds transport frame limit", {
bytes: oversized.bytes,
});
return oversized;
}
const totalObs = Object.values(observations).reduce(
(sum, arr) => sum + arr.length,
0,
);
logger.info("Export complete", {
sessions: paginatedSessions.length,
totalSessions: allSessions.length,
observations: totalObs,
memories: memories.length,
summaries: summaries.length,
collections: selection && [...selection],
});
return exportData;
},
);
sdk.registerFunction("mem::import",
async (data: {
exportData: ExportData;
strategy?: "merge" | "replace" | "skip";
}) => {
const kv = await pinGraphState(state);
if (
!data?.exportData ||
typeof data.exportData !== "object" ||
typeof (data.exportData as { version?: unknown }).version !== "string"
) {
return { success: false, error: "exportData with string version is required" };
}
const strategy = data.strategy || "merge";
const importData = data.exportData;
const supportedVersions = new Set(["0.3.0", "0.4.0", "0.5.0", "0.6.0", "0.6.1", "0.7.0", "0.7.2", "0.7.3", "0.7.4", "0.7.5", "0.7.6", "0.7.7", "0.7.9", "0.8.0", "0.8.1", "0.8.2", "0.8.3", "0.8.4", "0.8.5", "0.8.6", "0.8.7", "0.8.8", "0.8.9", "0.8.10", "0.8.11", "0.8.12", "0.8.13", "0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "0.9.5", "0.9.6", "0.9.7", "0.9.8", "0.9.9", "0.9.10", "0.9.11", "0.9.12", "0.9.13", "0.9.14", "0.9.15", "0.9.16", "0.9.17", "0.9.18", "0.9.19", "0.9.20", "0.9.21", "0.9.22", "0.9.23", "0.9.24", "0.9.25", "0.9.26", "0.9.27", "0.9.28", "0.9.29", "0.9.30", "0.9.31"]);
if (!supportedVersions.has(importData.version)) {
return {
success: false,
error: `Unsupported export version: ${importData.version}`,
};
}
const sessionPage = importData.pagination;
const partialSessions =
sessionPage !== undefined &&
((sessionPage.offset ?? 0) > 0 || sessionPage.hasMore === true);
if (strategy === "replace" && (importData.collectionPagination || partialSessions)) {
return {
success: false,
error:
"replace needs a complete export; this one is a page or a selection of collections",
};
}
const MAX_SESSIONS = 10_000;
const MAX_MEMORIES = 50_000;
const MAX_SUMMARIES = 10_000;
const MAX_TOTAL_OBSERVATIONS = 500_000;
const MAX_ACCESS_LOGS = 50_000;
if (!Array.isArray(importData.sessions)) {
return { success: false, error: "sessions must be an array" };
}
if (!Array.isArray(importData.memories)) {
return { success: false, error: "memories must be an array" };
}
if (!Array.isArray(importData.summaries)) {
return { success: false, error: "summaries must be an array" };
}
if (
typeof importData.observations !== "object" ||
importData.observations === null ||
Array.isArray(importData.observations)
) {
return { success: false, error: "observations must be an object" };
}
if (importData.sessions.length > MAX_SESSIONS) {
return {
success: false,
error: `Too many sessions (max ${MAX_SESSIONS})`,
};
}
if (importData.memories.length > MAX_MEMORIES) {
return {
success: false,
error: `Too many memories (max ${MAX_MEMORIES})`,
};
}
if (importData.summaries.length > MAX_SUMMARIES) {
return {
success: false,
error: `Too many summaries (max ${MAX_SUMMARIES})`,
};
}
const MAX_OBS_BUCKETS = 10_000;
const obsBuckets = Object.keys(importData.observations);
if (obsBuckets.length > MAX_OBS_BUCKETS) {
return {
success: false,
error: `Too many observation buckets (max ${MAX_OBS_BUCKETS})`,
};
}
let totalObservations = 0;
for (const [sessionId, obs] of Object.entries(importData.observations)) {
if (!Array.isArray(obs)) {
return { success: false, error: "observation values must be arrays" };
}
if (obs.length > MAX_IMPORT_OBS_PER_SESSION) {
return {
success: false,
error:
`Too many observations in session ${sessionId}: ${obs.length} ` +
`(max ${MAX_IMPORT_OBS_PER_SESSION}). MAX_OBS_PER_SESSION may be ` +
`configured above this import ceiling; lower it to at most ` +
`${MAX_IMPORT_OBS_PER_SESSION} or split the session before exporting.`,
};
}
totalObservations += obs.length;
}
if (totalObservations > MAX_TOTAL_OBSERVATIONS) {
return {
success: false,
error: `Too many total observations (max ${MAX_TOTAL_OBSERVATIONS})`,
};
}
try {
validateGraphRecords(importData.graphNodes ?? [], importData.graphEdges ?? []);
} catch (error) {
return { success: false, error: error instanceof Error ? error.message : String(error) };
}
const badRow = (rows: readonly unknown[], key: string) =>
rows.findIndex((row) => {
const id = row && typeof row === "object" ? (row as Record<string, unknown>)[key] : undefined;
return typeof id !== "string" || id.length === 0;
});
for (const [sessionId, obs] of Object.entries(importData.observations)) {
const index = badRow(obs, "id");
if (index >= 0) {
return { success: false, error: `observations.${sessionId}[${index}] must be an object with a string id` };
}
}
const rowKeys: Record<string, string> = { summaries: "sessionId", profiles: "project", accessLogs: "memoryId" };
for (const section of ["sessions", "memories", "summaries", "profiles", ...EXPORT_COLLECTIONS.map(([field]) => field)] as const) {
const rows = importData[section];
if (rows !== undefined && !Array.isArray(rows)) {
return { success: false, error: `${section} must be an array` };
}
const key = rowKeys[section] ?? "id";
const index = badRow(rows ?? [], key);
if (index >= 0) {
return { success: false, error: `${section}[${index}] must be an object with a string ${key}` };
}
}
if ((importData.accessLogs?.length ?? 0) > MAX_ACCESS_LOGS) {
return {
success: false,
error: `Too many access logs (max ${MAX_ACCESS_LOGS})`,
};
}
const stats = {
sessions: 0,
observations: 0,
memories: 0,
summaries: 0,
skipped: 0,
sourceTruncated: 0,
sourceOmitted: 0,
};
const replaceSessions = strategy === "replace" ? await kv.list<Session>(KV.sessions) : [];
const sessionKeys = [...obsBuckets, ...importData.sessions.map((session) => session.id),
...importData.summaries.map((summary) => summary.sessionId), ...replaceSessions.map((session) => session.id)]
.map((id) => `obs:${id}`);
const indexObs: CompressedObservation[] = [];
const indexMems: Memory[] = [];
const preflightError = await withKeyedLocks(sessionKeys, async () => {
const plannedObservations = new Map<string, CompressedObservation[]>();
for (const [sessionId, observations] of Object.entries(importData.observations)) {
const existing = strategy === "replace" ? [] : await kv.list<CompressedObservation>(KV.observations(sessionId));
const planned = budgetImportedObservationSources(observations.map((row) => ({
...row, origin: importOrigin(row.origin, row.timestamp),
})), existing, strategy);
if (!planned.success) return planned;
plannedObservations.set(sessionId, planned.observations);
stats.sourceTruncated += planned.sourceTruncated;
stats.sourceOmitted += planned.sourceOmitted;
}
if (strategy === "replace") {
const existing = replaceSessions;
// Collect observation deletes across all sessions, then run them in
// one bounded pass: a runChunked nested inside a runChunked callback
// multiplies in-flight deletes to chunk-size squared.
const obsDeletes: Array<{ sessionId: string; obsId: string; captureKey?: string }> = [];
await runChunked(existing, async (session) => {
await kv.delete(KV.sessions, session.id);
await kv.delete(KV.summaryRepairs, session.id);
await removeSessionFromProjectIndex(
kv,
session.project,
session.id,
).catch(() => {});
const obs = await kv
.list<CompressedObservation>(KV.observations(session.id))
.catch(() => []);
for (const o of obs) {
obsDeletes.push({ sessionId: session.id, obsId: o.id, captureKey: o.captureKey });
}
});
await runChunked(obsDeletes, async (d) => {
await markCaptureEventDeleted(kv, { id: d.obsId, sessionId: d.sessionId, captureKey: d.captureKey });
await kv.delete(KV.observations(d.sessionId), d.obsId);
await unindexObservationSession(kv, d.obsId).catch(() => {});
});
await runChunked(await kv.list<Memory>(KV.memories), (m) =>
kv.delete(KV.memories, m.id),
);
await runChunked(
await kv.list<SessionSummary>(KV.summaries),
(s) => kv.delete(KV.summaries, s.sessionId),
);
await runChunked(await kv.list<Action>(KV.actions).catch(() => []), (a) =>
kv.delete(KV.actions, a.id),
);
await runChunked(
await kv.list<ActionEdge>(KV.actionEdges).catch(() => []),
(e) => kv.delete(KV.actionEdges, e.id),
);
await runChunked(
await kv.list<Routine>(KV.routines).catch(() => []),
(r) => kv.delete(KV.routines, r.id),
);
await runChunked(
await kv.list<Signal>(KV.signals).catch(() => []),
(s) => kv.delete(KV.signals, s.id),
);
await runChunked(
await kv.list<Checkpoint>(KV.checkpoints).catch(() => []),
(c) => kv.delete(KV.checkpoints, c.id),
);
await runChunked(
await kv.list<Sentinel>(KV.sentinels).catch(() => []),
(s) => kv.delete(KV.sentinels, s.id),
);
await runChunked(
await kv.list<Sketch>(KV.sketches).catch(() => []),
(s) => kv.delete(KV.sketches, s.id),
);
await runChunked(
await kv.list<Crystal>(KV.crystals).catch(() => []),
(c) => kv.delete(KV.crystals, c.id),
);
await runChunked(
await kv.list<Facet>(KV.facets).catch(() => []),
(f) => kv.delete(KV.facets, f.id),
);
await runChunked(
await kv.list<Lesson>(KV.lessons).catch(() => []),
(l) => kv.delete(KV.lessons, l.id),
);
resetLessonIndex();
await runChunked(
await kv.list<Insight>(KV.insights).catch(() => []),
(i) => kv.delete(KV.insights, i.id),
);
await runChunked(
await kv.list<{ id: string }>(KV.semantic).catch(() => []),
(s) => kv.delete(KV.semantic, s.id),
);
await runChunked(
await kv.list<{ id: string }>(KV.procedural).catch(() => []),
(p) => kv.delete(KV.procedural, p.id),
);
await runChunked(
await kv.list<ProjectProfile>(KV.profiles).catch(() => []),
(profile) => kv.delete(KV.profiles, profile.project),
);
await runChunked(
await kv.list<AccessLogExport>(KV.accessLog).catch(() => []),
(a) => kv.delete(KV.accessLog, a.memoryId),
);
}
await runChunked(importData.sessions, async (session) => {
if (strategy === "skip") {
const existing = await kv.get<Session>(KV.sessions, session.id).catch(() => null);
if (existing) {
stats.skipped++;
return;
}
}
await kv.set(KV.sessions, session.id, session);
stats.sessions++;
await addSessionToProjectIndex(kv, session.project, {
id: session.id,
startedAt: session.startedAt,
...(session.agentId ? { agentId: session.agentId } : {}),
}).catch(() => {});
});
for (const [sessionId, obs] of plannedObservations) {
await runChunked(obs, async (o) => {
if (strategy === "skip") {
const existing = await kv
.get<CompressedObservation>(KV.observations(sessionId), o.id)
.catch(() => null);
if (existing) {
stats.skipped++;
return;
}
}
await kv.set(KV.observations(sessionId), o.id, o);
await indexObservationSession(kv, o.id, sessionId).catch(() => {});
stats.observations++;
indexObs.push(o);
});
}
await runChunked(importData.summaries, async (summary) => {
if (strategy === "skip") {
const existing = await kv
.get<SessionSummary>(KV.summaries, summary.sessionId)
.catch(() => null);
if (existing) {
stats.skipped++;
return;
}
}
await kv.set(KV.summaries, summary.sessionId, summary);
stats.summaries++;
});
return null;
});
if (preflightError) return preflightError;
await runChunked(importData.memories, async (memory) => {
if (strategy === "skip") {
const existing = await kv
.get<Memory>(KV.memories, memory.id)
.catch(() => null);
if (existing) {
stats.skipped++;
return;
}
}
// Older exports + hand-edited dumps can omit this field.
if (!Array.isArray(memory.sessionIds)) {
memory.sessionIds = [];
}
memory.origin = importOrigin(memory.origin, memory.createdAt);
await kv.set(KV.memories, memory.id, memory);
stats.memories++;
indexMems.push(memory);
});
if (strategy === "replace" || importData.graphNodes || importData.graphEdges) {
const graph = await persistGraphRecords(kv, importData.graphNodes ?? [], importData.graphEdges ?? [], {
mode: strategy === "skip" ? "skip" : "merge",
replace: strategy === "replace",
});
stats.skipped += graph.nodesSkipped + graph.edgesSkipped;
}
if (importData.semanticMemories) {
await runChunked(importData.semanticMemories, async (sem) => {
if (strategy === "skip") {
const existing = await kv.get(KV.semantic, sem.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.semantic, sem.id, sem);
});
}
if (importData.proceduralMemories) {
await runChunked(importData.proceduralMemories, async (proc) => {
if (strategy === "skip") {
const existing = await kv.get(KV.procedural, proc.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.procedural, proc.id, proc);
});
}
if (importData.profiles) {
await runChunked(importData.profiles, async (profile) => {
if (strategy === "skip") {
const existing = await kv
.get<ProjectProfile>(KV.profiles, profile.project)
.catch(() => null);
if (existing) {
stats.skipped++;
return;
}
}
await kv.set(KV.profiles, profile.project, profile);
});
}
if (importData.actions) {
await runChunked(importData.actions, async (action) => {
if (strategy === "skip") {
const existing = await kv.get(KV.actions, action.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.actions, action.id, action);
});
}
if (importData.actionEdges) {
await runChunked(importData.actionEdges, async (edge) => {
if (strategy === "skip") {
const existing = await kv.get(KV.actionEdges, edge.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.actionEdges, edge.id, edge);
});
}
if (importData.routines) {
await runChunked(importData.routines, async (routine) => {
if (strategy === "skip") {
const existing = await kv.get(KV.routines, routine.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.routines, routine.id, routine);
});
}
if (importData.signals) {
await runChunked(importData.signals, async (signal) => {
if (strategy === "skip") {
const existing = await kv.get(KV.signals, signal.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.signals, signal.id, signal);
});
}
if (importData.checkpoints) {
await runChunked(importData.checkpoints, async (checkpoint) => {
if (strategy === "skip") {
const existing = await kv.get(KV.checkpoints, checkpoint.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.checkpoints, checkpoint.id, checkpoint);
});
}
if (importData.sentinels) {
await runChunked(importData.sentinels, async (sentinel) => {
if (strategy === "skip") {
const existing = await kv.get(KV.sentinels, sentinel.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.sentinels, sentinel.id, sentinel);
});
}
if (importData.sketches) {
await runChunked(importData.sketches, async (sketch) => {
if (strategy === "skip") {
const existing = await kv.get(KV.sketches, sketch.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.sketches, sketch.id, sketch);
});
}
if (importData.crystals) {
await runChunked(importData.crystals, async (crystal) => {
if (strategy === "skip") {
const existing = await kv.get(KV.crystals, crystal.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.crystals, crystal.id, crystal);
});
}
if (importData.facets) {
await runChunked(importData.facets, async (facet) => {
if (strategy === "skip") {
const existing = await kv.get(KV.facets, facet.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.facets, facet.id, facet);
});
}
if (importData.lessons) {
await runChunked(importData.lessons, async (lesson) => {
if (strategy === "skip") {
const existing = await kv.get(KV.lessons, lesson.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.lessons, lesson.id, lesson);
});
resetLessonIndex();
}
if (importData.insights) {
await runChunked(importData.insights, async (insight) => {
if (strategy === "skip") {
const existing = await kv.get(KV.insights, insight.id).catch(() => null);
if (existing) { stats.skipped++; return; }
}
await kv.set(KV.insights, insight.id, insight);
});
}
if (importData.accessLogs) {
const memoryIds = new Set<string>(
importData.memories.map((m) => m.id),
);
await runChunked(importData.accessLogs, async (raw) => {
const log = normalizeAccessLog(raw);
if (!log.memoryId || !memoryIds.has(log.memoryId)) return;
if (strategy === "skip") {
const existing = await kv
.get(KV.accessLog, log.memoryId)
.catch(() => null);
if (existing) {
stats.skipped++;
return;
}
}
await kv.set(KV.accessLog, log.memoryId, log);
});
}
// Imported rows are now in KV but invisible to search: the boot
// rebuild gate only fires when BM25 is empty, so on any existing
// install (non-empty persisted index) imported observations and
// memories never surface via mem::search / smart-search until a
// manual rebuild. Add them to BM25 (synchronous) and enqueue the
// vector embeddings in batches (one embedBatch call per chunk)
// rather than one giant Promise.all over 500k docs. Indexing
// failures are logged, not fatal — the KV writes already committed
// and the restart rebuild is the backstop.
try {
await indexRecords(indexObs, indexMems);
} catch (err) {
logger.warn("Import indexing failed; restart rebuild will recover", {
error: err instanceof Error ? err.message : String(err),
});
}
logger.info("Import complete", { strategy, ...stats });
await recordAudit(kv, "import", "mem::import", [], {
strategy,
stats,
});
return { success: true, strategy, ...stats };
},
);
}