Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

feat(replay): auto-derive lessons + crystals from imported JSONL

Rohit Ghumareae12890cfd31
diff --git a/src/functions/replay.ts b/src/functions/replay.ts
index 49fbc7b4..08f083f3 100644
--- a/src/functions/replay.ts
+++ b/src/functions/replay.ts
@@ -4,6 +4,8 @@ import { resolve, join } from "node:path";
import type { ISdk } from "iii-sdk";
import type {
CompressedObservation,
+ Crystal,
+ Lesson,
RawObservation,
Session,
} from "../types.js";
@@ -56,6 +58,105 @@ function rawFromCompressed(obs: CompressedObservation): RawObservation {
};
}
+const LESSON_PATTERNS: RegExp[] = [
+ /\b(always|never|don'?t|do not|make sure|remember to|note:|caveat:|warning:)\b[^.\n]{10,200}[.!\n]/gi,
+ /\b(prefer|avoid)\s[^.\n]{10,200}[.!\n]/gi,
+];
+
+async function deriveCrystalAndLessons(
+ kv: StateKV,
+ sessionId: string,
+ project: string,
+ rawObs: RawObservation[],
+ compressed: CompressedObservation[],
+ firstPrompt: string | undefined,
+): Promise<void> {
+ if (rawObs.length === 0) return;
+ const createdAt = new Date().toISOString();
+
+ const files = new Set<string>();
+ const tools = new Set<string>();
+ for (const c of compressed) {
+ for (const f of c.files || []) files.add(f);
+ if (c.type && c.type !== "conversation" && c.title) tools.add(c.title);
+ }
+
+ const assistantTexts: string[] = [];
+ const userPrompts: string[] = [];
+ for (const r of rawObs) {
+ if (typeof r.assistantResponse === "string" && r.assistantResponse.trim()) {
+ assistantTexts.push(r.assistantResponse);
+ }
+ if (typeof r.userPrompt === "string" && r.userPrompt.trim()) {
+ userPrompts.push(r.userPrompt);
+ }
+ }
+
+ const lessonMatches = new Map<string, string>();
+ for (const text of assistantTexts.concat(userPrompts).slice(0, 200)) {
+ for (const pat of LESSON_PATTERNS) {
+ pat.lastIndex = 0;
+ let m: RegExpExecArray | null;
+ while ((m = pat.exec(text)) !== null && lessonMatches.size < 40) {
+ const snippet = m[0].replace(/\s+/g, " ").trim();
+ if (snippet.length >= 20 && snippet.length <= 220) {
+ const key = snippet.toLowerCase();
+ if (!lessonMatches.has(key)) lessonMatches.set(key, snippet);
+ }
+ }
+ }
+ }
+
+ const lessonEntries = Array.from(lessonMatches.values()).slice(0, 20);
+ const lessonIds: string[] = [];
+ for (const content of lessonEntries) {
+ const lessonId = generateId("lesson");
+ const lesson: Lesson = {
+ id: lessonId,
+ content,
+ context: firstPrompt || project,
+ confidence: 0.4,
+ reinforcements: 0,
+ source: "consolidation",
+ sourceIds: [sessionId],
+ project,
+ tags: ["auto-import"],
+ createdAt,
+ updatedAt: createdAt,
+ decayRate: 0.05,
+ };
+ try {
+ await kv.set(KV.lessons, lessonId, lesson);
+ lessonIds.push(lessonId);
+ } catch {}
+ }
+
+ const crystalId = generateId("crystal");
+ const narrativePreview = firstPrompt
+ ? firstPrompt.slice(0, 300)
+ : compressed
+ .slice(0, 5)
+ .map((c) => c.narrative || c.title)
+ .filter(Boolean)
+ .join(" · ")
+ .slice(0, 300);
+
+ const crystal: Crystal = {
+ id: crystalId,
+ narrative: narrativePreview || `Session ${sessionId.slice(0, 12)} (${rawObs.length} observations)`,
+ keyOutcomes: Array.from(tools).slice(0, 8),
+ filesAffected: Array.from(files).slice(0, 20),
+ lessons: lessonIds,
+ sourceActionIds: [],
+ sessionId,
+ project,
+ createdAt,
+ };
+ try {
+ await kv.set(KV.crystals, crystalId, crystal);
+ } catch {}
+}
+
function isRawShape(o: unknown): o is RawObservation {
if (!o || typeof o !== "object") return false;
const r = o as Record<string, unknown>;
@@ -237,15 +338,26 @@ export function registerReplayFunctions(sdk: ISdk, kv: StateKV): void {
}
const searchIndex = getSearchIndex();
+ const compressed: CompressedObservation[] = [];
await Promise.all(
parsed.observations.map(async (obs) => {
const synthetic = buildSyntheticCompression(obs);
+ compressed.push(synthetic);
await kv.set(KV.observations(parsed.sessionId), obs.id, synthetic);
searchIndex.add(synthetic);
}),
);
observationCount += parsed.observations.length;
sessionIds.push(parsed.sessionId);
+
+ await deriveCrystalAndLessons(
+ kv,
+ parsed.sessionId,
+ parsed.project,
+ parsed.observations,
+ compressed,
+ firstPrompt,
+ );
}
await safeAudit(kv, "import", "mem::replay::import-jsonl", sessionIds, {