Skip to content

#1 Persistent memory for AI coding agents based on real-world benchmarks

fix(import-jsonl): ignore timestampless entries for session times and make re-import idempotent (#1526)

YuXuan Tayb3164b582601
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c7e8f497..d7045cec 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
### Fixed
+- **JSONL import keeps real session times and no longer duplicates on re-import.** `parseJsonlText` took `startedAt`/`endedAt` from the first and last line even when it was a timestampless metadata entry (`last-prompt`, `cost-state`), so the session got the import time. It now uses only entries that carry a timestamp. Observation ids are derived from the entry `uuid` (or session id and line position when absent) instead of a random id, so importing the same transcript again overwrites the same rows, and `observationCount` counts only rows that were new. Rows imported by earlier versions keep their random ids, so the first re-import after upgrading adds one last copy.
- **Session summaries cover the whole session again** (#745). 0.9.30 completed a session on the first per-turn Stop and ignored every later end, so summaries and graph extraction stopped at turn 1. `mem::observe` now reopens a completed session when new observations arrive, the Stop hook sends `final: false` so the session stays open, and an idle finalizer completes it once nothing has happened for `AGENTMEMORY_FINALIZE_IDLE_MS` (default 2 minutes). SessionEnd, the CLI and the OpenCode plugin send `final: true` and complete at once; a request without the flag behaves as before. The sweep summarizes a session before marking it abandoned (#737).
- **The Devin plugin captures sessions.** A plugin install loaded `hooks/hooks.json`, whose capitalized Claude Code matchers never match Devin's tool names and whose extra events Devin rejects, so capture never fired. `plugin/hooks.json` now carries the six Devin hooks with lowercase matchers, and Devin loads it first. Verified on Devin CLI 3000.11.3: one session recorded session start, prompt, tool use and session end.
- **Summaries are incremental.** `mem::summarize` keeps a watermark (`coveredThrough`) and summarizes only the observations captured since the last run, merging them into the stored summary with one reduce call. A session end with nothing new costs no LLM call, and the reflect, graph and skill fan-out is skipped for it.
diff --git a/src/functions/replay.ts b/src/functions/replay.ts
index 6e063240..ac80a46e 100644
--- a/src/functions/replay.ts
+++ b/src/functions/replay.ts
@@ -474,7 +474,9 @@ export function registerReplayFunctions(sdk: IIIClient, kv: StateKV): void {
firstPrompt,
};
sessionRow.id ||= parsed.sessionId;
- sessionRow.observationCount = (sessionRow.observationCount || 0) + parsed.observations.length;
+ const knownIds = new Set(existingRows.map((row) => row.id));
+ const added = planned.observations.filter((obs) => !knownIds.has(obs.id)).length;
+ sessionRow.observationCount = (sessionRow.observationCount || 0) + added;
if (parsed.endedAt > (sessionRow.endedAt || "")) sessionRow.endedAt = parsed.endedAt;
if (sessionRow.status === "active") sessionRow.status = "completed";
if (!sessionRow.tags?.includes("jsonl-import")) sessionRow.tags = [...(sessionRow.tags || []), "jsonl-import"];
@@ -486,7 +488,7 @@ export function registerReplayFunctions(sdk: IIIClient, kv: StateKV): void {
const failure = results.find((result) => result.status === "rejected");
if (failure) throw failure.reason;
}
- return { ...planned, sessionRow };
+ return { ...planned, sessionRow, added };
});
if (!committed.success) return { ...committed, imported: sessionIds.length, sessionIds, observations: observationCount };
const { sessionRow, observations: compressed } = committed;
@@ -507,7 +509,7 @@ export function registerReplayFunctions(sdk: IIIClient, kv: StateKV): void {
error: err instanceof Error ? err.message : String(err),
});
}
- observationCount += parsed.observations.length;
+ observationCount += committed.added;
sessionIds.push(parsed.sessionId);
await deriveCrystalAndLessons(
diff --git a/src/replay/jsonl-parser.ts b/src/replay/jsonl-parser.ts
index c5f5ee4c..745c2c79 100644
--- a/src/replay/jsonl-parser.ts
+++ b/src/replay/jsonl-parser.ts
@@ -2,7 +2,7 @@ import { existsSync, realpathSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { basename, dirname, resolve } from "node:path";
import type { HookType, RawObservation } from "../types.js";
-import { generateId } from "../state/schema.js";
+import { fingerprintId, generateId } from "../state/schema.js";
interface JsonlEntry {
type?: string;
@@ -143,12 +143,17 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed
const observations: RawObservation[] = [];
- for (const entry of entries) {
+ for (const [index, entry] of entries.entries()) {
+ let produced = 0;
+ const nextId = () =>
+ fingerprintId("obs", `${entry.uuid ?? `${sessionId}:${index}`}:${produced++}`);
if (entry.sessionId && !sessionId) sessionId = entry.sessionId;
if (typeof entry.cwd === "string" && entry.cwd.trim() && !cwd) cwd = entry.cwd;
- const ts = entry.timestamp || new Date().toISOString();
- if (!firstTs) firstTs = ts;
- lastTs = ts;
+ if (entry.timestamp) {
+ if (!firstTs) firstTs = entry.timestamp;
+ lastTs = entry.timestamp;
+ }
+ const ts = entry.timestamp || lastTs || new Date().toISOString();
const role = entry.message?.role;
const content = entry.message?.content;
@@ -158,7 +163,7 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed
if (toolResults.length > 0) {
for (const result of toolResults) {
observations.push({
- id: generateId("obs"),
+ id: nextId(),
sessionId: sessionId || "imported",
timestamp: ts,
hookType: (result.isError ? "post_tool_failure" : "post_tool_use") as HookType,
@@ -172,7 +177,7 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed
const text = toText(content);
if (text.trim().length > 0) {
observations.push({
- id: generateId("obs"),
+ id: nextId(),
sessionId: sessionId || "imported",
timestamp: ts,
hookType: "prompt_submit" as HookType,
@@ -186,7 +191,7 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed
const tools = extractToolUses(content);
if (text.trim().length > 0) {
observations.push({
- id: generateId("obs"),
+ id: nextId(),
sessionId: sessionId || "imported",
timestamp: ts,
hookType: "stop" as HookType,
@@ -196,7 +201,7 @@ export function parseJsonlText(text: string, fallbackSessionId?: string): Parsed
}
for (const tool of tools) {
observations.push({
- id: generateId("obs"),
+ id: nextId(),
sessionId: sessionId || "imported",
timestamp: ts,
hookType: "pre_tool_use" as HookType,
diff --git a/test/replay-write-batching.test.ts b/test/replay-write-batching.test.ts
index dce0bc9d..a7175125 100644
--- a/test/replay-write-batching.test.ts
+++ b/test/replay-write-batching.test.ts
@@ -185,3 +185,44 @@ describe("JSONL observation write batching", () => {
}
});
});
+
+describe("JSONL re-import", () => {
+ it("overwrites the same rows and counts only new observations", async () => {
+ const folder = mkdtempSync(join(__dirname, "replay-reimport-"));
+ process.env.AGENTMEMORY_IMPORT_ROOT = folder;
+ const path = join(folder, "session.jsonl");
+ const sessionId = "reimport-session";
+ const line = (i: number) => JSON.stringify({
+ type: "user", uuid: `entry-${i}`, sessionId, timestamp: "2026-09-30T00:00:00Z",
+ message: { role: "user", content: `record ${i}` },
+ });
+ const store = new Map<string, Map<string, unknown>>();
+ const kv = {
+ get: async (bucket: string, key: string) => store.get(bucket)?.get(key) ?? null,
+ list: async (bucket: string) => Array.from(store.get(bucket)?.values() ?? []),
+ set: async (bucket: string, key: string, value: unknown) => {
+ if (!store.has(bucket)) store.set(bucket, new Map());
+ store.get(bucket)!.set(key, value);
+ return value;
+ },
+ };
+ let importJsonl: (data: { path: string }) => Promise<unknown> = async () => undefined;
+ const sdk = {
+ registerFunction: (id: string, handler: typeof importJsonl) => {
+ if (id === "mem::replay::import-jsonl") importJsonl = handler;
+ },
+ };
+ registerReplayFunctions(sdk as never, kv as never);
+ try {
+ writeFileSync(path, [0, 1, 2].map(line).join("\n"));
+ expect(await importJsonl({ path })).toMatchObject({ success: true, observations: 3 });
+ expect(await importJsonl({ path })).toMatchObject({ success: true, observations: 0 });
+ writeFileSync(path, [0, 1, 2, 3].map(line).join("\n"));
+ expect(await importJsonl({ path })).toMatchObject({ success: true, observations: 1 });
+ expect(store.get(KV.observations(sessionId))?.size).toBe(4);
+ expect((store.get(KV.sessions)?.get(sessionId) as { observationCount: number }).observationCount).toBe(4);
+ } finally {
+ rmSync(folder, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/test/replay.test.ts b/test/replay.test.ts
index 1ebc0d91..6747a4e4 100644
--- a/test/replay.test.ts
+++ b/test/replay.test.ts
@@ -167,3 +167,37 @@ describe("rawFromCompressed", () => {
});
});
+
+describe("parseJsonlText metadata lines and re-import", () => {
+ const jsonl = [
+ { type: "last-prompt" },
+ { type: "user", sessionId: "s1", timestamp: "2026-10-09T09:12:38Z", message: { role: "user", content: "hi" } },
+ { type: "assistant", sessionId: "s1", timestamp: "2026-10-09T09:54:34Z", message: { role: "assistant", content: "hello" } },
+ { type: "cost-state" },
+ ].map((e) => JSON.stringify(e)).join("\n");
+
+ it("ignores timestampless entries for startedAt/endedAt", () => {
+ const out = parseJsonlText(jsonl);
+ expect(out.startedAt).toBe("2026-10-09T09:12:38Z");
+ expect(out.endedAt).toBe("2026-10-09T09:54:34Z");
+ });
+
+ it("derives the same ids on re-parse and distinct ids for files sharing a session", () => {
+ const entry = (uuid: string, text: string) => JSON.stringify({
+ type: "user", uuid, sessionId: "s1", timestamp: "2026-10-09T09:12:38Z", message: { role: "user", content: text },
+ });
+ const a = `${entry("a0", "one")}\n${entry("a1", "two")}`;
+ const b = `${entry("b0", "three")}\n${entry("b1", "four")}`;
+ const ids = (t: string) => parseJsonlText(t).observations.map((o) => o.id);
+ expect(ids(a)).toEqual(ids(a));
+ expect(new Set([...ids(a), ...ids(b)]).size).toBe(4);
+ });
+
+ it("gives a timestampless entry the previous entry's time", () => {
+ const out = parseJsonlText([
+ JSON.stringify({ type: "user", sessionId: "s1", timestamp: "2026-10-09T09:12:38Z", message: { role: "user", content: "a" } }),
+ JSON.stringify({ type: "assistant", sessionId: "s1", message: { role: "assistant", content: "b" } }),
+ ].join("\n"));
+ expect(out.observations.map((o) => o.timestamp)).toEqual(["2026-10-09T09:12:38Z", "2026-10-09T09:12:38Z"]);
+ });
+});