Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* transcripts-ingest e2e (PGLite) — cathedral-4.
*
* Pins the import lane end-to-end against a real embedded engine:
* cross-harness round-trip, dry-run zero-writes, idempotent re-runs,
* redaction-before-write, part splitting under the embed-skip threshold,
* the DANGEROUS TRANSITIONS (split→shrink stale-part deletion), since/limit
* clean-scan semantics, and the putRawData zero-row parity fix.
*
* R3/R4: engine in beforeAll, disconnect in afterAll; state reset per test.
*
* Lane: unit. Run: `bun test test/transcripts-ingest-pglite.test.ts`. Moved from test/e2e/
* by the 2026-09 lane-move pilot (PGLite-only, no DATABASE_URL); see
* docs/test-audit/2026-09-29/implementation/lane-pilot.md.
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { copyFileSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { runTranscriptsIngest } from '../src/core/transcripts/ingest.ts';
import { runIngestFacts } from '../src/core/transcripts/ingest-facts.ts';
import {
buildStatusRows,
discoverTranscriptFiles,
indexImportedSessions,
} from '../src/core/transcripts/discover.ts';
import type { HarnessRoot } from '../src/core/transcripts/detect.ts';
import { buildTranscriptSlug } from '../src/core/transcripts/types.ts';
import { buildHermesFixture } from './fixtures/transcripts/hermes-fixture-builder.ts';
import { PROVIDER_KEY_SHAPES } from './helpers/provider-keys.ts';
const CODEX_SLUG = buildTranscriptSlug('codex', '2026-08-02T09:00:00.000Z', {
sessionId: 'rollout-1', // payload.id — the per-thread id (#4981)
});
const AGENT_SLUG = buildTranscriptSlug('openclaw', '2026-08-03T14:00:00.000Z', {
sessionId: 'agent-fixture-session-1',
});
const CODEX_FIXTURE = join(import.meta.dir, 'fixtures', 'transcripts', 'codex-rollout.jsonl');
const AGENT_FIXTURE = join(import.meta.dir, 'fixtures', 'transcripts', 'agent-session.jsonl');
const CLAUDE_CODE_FIXTURE = join(
import.meta.dir,
'fixtures',
'conversation-formats',
'claude-code.jsonl',
);
const CHATGPT_FIXTURE = join(
import.meta.dir,
'fixtures',
'transcripts',
'chatgpt-conversations.json',
);
const CLAUDE_EXPORT_FIXTURE = join(
import.meta.dir,
'fixtures',
'transcripts',
'claude-export.json',
);
const GROK_FIXTURE = join(
import.meta.dir,
'fixtures',
'transcripts',
'grok-session',
'chat_history.jsonl',
);
let engine: PGLiteEngine;
let tmp: string;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
});
afterAll(async () => {
await engine.disconnect();
});
beforeEach(async () => {
await resetPgliteState(engine);
tmp = mkdtempSync(join(tmpdir(), 'gb-ingest-e2e-'));
});
afterEach(() => {
rmSync(tmp, { recursive: true, force: true });
});
const NO_PATTERNS = { userPatternsPath: '/nonexistent-patterns.txt' };
// Synthetic AWS-shaped token, built at runtime so the literal never lands in
// committed bytes (the pre-push credential guard would flag it — correctly).
const PLANTED_KEY = ['AKIA', 'ABCDEFGHIJKLMNOP'].join('');
function baseOpts(paths: string[], extra: Record<string, unknown> = {}) {
return { paths, sourceId: 'default', ...NO_PATTERNS, ...extra };
}
/** Synthetic openclaw-format session with N large messages. */
function writeBigAgentSession(dir: string, id: string, messageCount: number): string {
const lines: string[] = [
JSON.stringify({ type: 'session', version: 3, id, timestamp: '2026-08-10T08:00:00.000Z', cwd: '/tmp' }),
];
const filler = 'lorem widget fact '.repeat(Math.ceil(3_900 / 18));
for (let i = 0; i < messageCount; i++) {
lines.push(
JSON.stringify({
type: 'message',
id: `m-${i}`,
timestamp: `2026-08-10T08:${String(Math.floor(i / 60)).padStart(2, '0')}:${String(i % 60).padStart(2, '0')}.000Z`,
message: {
role: i % 2 === 0 ? 'user' : 'assistant',
timestamp: `2026-08-10T08:${String(Math.floor(i / 60)).padStart(2, '0')}:${String(i % 60).padStart(2, '0')}.000Z`,
content: [{ type: 'text', text: `marker-${i} ${filler}` }],
},
}),
);
}
const p = join(dir, `${id}.jsonl`);
writeFileSync(p, lines.join('\n') + '\n');
return p;
}
describe('cross-harness round-trip', () => {
test('codex + openclaw fixtures land as conversation pages in one source', async () => {
const r = await runTranscriptsIngest(engine, baseOpts([CODEX_FIXTURE, AGENT_FIXTURE]));
expect(r.sessionsImported).toBe(2);
expect(r.pages.imported).toBe(2);
// The committed fixtures carry deliberate malformed tail lines — a
// possibly-dropped record must freeze the watermark, so this is NOT a
// clean scan (pristine-file cleanliness is pinned in the since/limit
// suite below).
expect(r.cleanScan).toBe(false);
expect(r.erroredFiles).toBe(0);
const codexPage = await engine.getPage(CODEX_SLUG, {
sourceId: 'default',
});
expect(codexPage).not.toBeNull();
expect(codexPage!.type).toBe('conversation');
expect(codexPage!.compiled_truth).toContain('fund-a led the widget-co seed');
expect(codexPage!.compiled_truth).not.toContain('PREAMBLE-ONLY-TEXT');
const agentPage = await engine.getPage(AGENT_SLUG, {
sourceId: 'default',
});
expect(agentPage).not.toBeNull();
expect(agentPage!.compiled_truth).toContain('acme-seed memo');
// Cross-harness continuity substrate: both sessions in ONE brain source.
const fm = agentPage!.frontmatter as Record<string, any>;
expect(fm.transcript_import.harness).toBe('openclaw');
expect(fm.transcript_import.session_id).toBe('agent-fixture-session-1');
expect(fm.date).toBe('2026-08-03');
// Session metadata rode putRawData onto the base page.
const raw = await engine.getRawData(agentPage!.slug, undefined, { sourceId: 'default' });
expect(raw.length).toBeGreaterThan(0);
expect((raw[0].data as Record<string, unknown>).session_id).toBe('agent-fixture-session-1');
});
});
describe('--max-bytes reaches the adapters', () => {
// Pins the WIRING, not the adapter. ingest.ts used to call
// adapter.parse(path) with no opts at all, so maxBytes could never arrive
// from the import lane — and an adapter-level test cannot see that, because
// it calls codexAdapter.parse() directly. Revert the ingest.ts threading and
// this test is the only thing in the suite that fails.
test('a rollout over the budget drops mid-file turns but keeps head identity and the tail', async () => {
const p = join(tmp, 'oversized-rollout.jsonl');
const line = (payload: Record<string, unknown>, ts: string) =>
JSON.stringify({ type: 'event_msg', timestamp: ts, payload });
const filler = JSON.stringify({
type: 'response_item',
timestamp: '2026-01-01T00:00:01.000Z',
payload: { type: 'reasoning', content: 'x'.repeat(4000) },
});
const lines = [
JSON.stringify({
type: 'session_meta',
timestamp: '2026-01-01T00:00:00.000Z',
payload: { session_id: 'wired-sess-1', cwd: '/w', cli_version: '1.2.3' },
}),
// Past the 16KB head window (budget/4) and far from the tail, so a
// correctly-budgeted read MUST drop it. Without the ingest→adapter
// wiring the whole 1.6MB file is read and this turn survives — which is
// exactly what makes this test pin the wiring rather than the adapter.
...Array(8).fill(filler),
line({ type: 'user_message', message: 'MIDDLE_TURN_MUST_BE_DROPPED' }, '2026-01-01T00:01:00.000Z'),
...Array(400).fill(filler),
line({ type: 'user_message', message: 'NEWEST_TURN_SURVIVES' }, '2026-01-01T00:09:00.000Z'),
];
writeFileSync(p, lines.join('\n') + '\n');
const r = await runTranscriptsIngest(engine, baseOpts([p], { maxBytes: 64 * 1024 }));
expect(r.erroredFiles).toBe(0);
expect(r.sessionsImported).toBe(1);
const pages = await engine.listPages({ type: 'conversation', sourceId: 'default', limit: 10 });
expect(pages).toHaveLength(1);
const page = await engine.getPage(pages[0].slug, { sourceId: 'default' });
expect(page).not.toBeNull();
expect(page!.compiled_truth).toContain('NEWEST_TURN_SURVIVES');
expect(page!.compiled_truth).not.toContain('MIDDLE_TURN_MUST_BE_DROPPED');
});
// A truncated read leaves seam partials in skippedLines, which freezes the
// since:last watermark. Pre-existing behaviour (the old throw froze it too),
// pinned here so a future change to seam accounting is a deliberate one.
test('a truncated read is never a clean scan', async () => {
const p = join(tmp, 'oversized-rollout-2.jsonl');
const meta = JSON.stringify({
type: 'session_meta',
timestamp: '2026-01-01T00:00:00.000Z',
payload: { session_id: 'wired-sess-2', cwd: '/w' },
});
const filler = JSON.stringify({
type: 'response_item',
timestamp: '2026-01-01T00:00:01.000Z',
payload: { type: 'reasoning', content: 'y'.repeat(4000) },
});
const newest = JSON.stringify({
type: 'event_msg',
timestamp: '2026-01-01T00:09:00.000Z',
payload: { type: 'user_message', message: 'tail turn' },
});
writeFileSync(p, [meta, ...Array(400).fill(filler), newest].join('\n') + '\n');
const r = await runTranscriptsIngest(engine, baseOpts([p], { maxBytes: 64 * 1024 }));
expect(r.cleanScan).toBe(false);
});
});
describe('dry-run', () => {
test('writes NOTHING — no pages, no raw data — and never advances watermarks', async () => {
const r = await runTranscriptsIngest(engine, baseOpts([CODEX_FIXTURE], { dryRun: true }));
expect(r.pages.planned).toBe(1);
expect(r.pages.imported).toBe(0);
expect(r.cleanScan).toBe(false); // dry-runs must not advance the watermark
const pages = await engine.listPages({ type: 'conversation', sourceId: 'default', limit: 10 });
expect(pages).toHaveLength(0);
});
test('--limit truncates the dry-run preview the same way it truncates the real run (#4762)', async () => {
// Two single-part fixtures, limit 1: the preview must cover ONE session,
// not the whole backlog. The second session is reached (sessionsSeen 2)
// and then cut by the same gate the write path uses.
const r = await runTranscriptsIngest(
engine,
baseOpts([CODEX_FIXTURE, AGENT_FIXTURE], { dryRun: true, limit: 1 }),
);
expect(r.sessionsSeen).toBe(2);
expect(r.pages.planned).toBe(1);
expect(r.pages.imported).toBe(0);
expect(r.cleanScan).toBe(false);
const pages = await engine.listPages({ type: 'conversation', sourceId: 'default', limit: 10 });
expect(pages).toHaveLength(0);
});
});
describe('idempotency', () => {
test('second run hash-skips every page; slugsTouched still includes them (facts re-runs)', async () => {
const r1 = await runTranscriptsIngest(engine, baseOpts([CODEX_FIXTURE, AGENT_FIXTURE]));
expect(r1.pages.imported).toBe(2);
const r2 = await runTranscriptsIngest(engine, baseOpts([CODEX_FIXTURE, AGENT_FIXTURE]));
expect(r2.pages.imported).toBe(0);
expect(r2.pages.skipped).toBe(2);
// The facts lane must see hash-skipped slugs too (CX14).
expect(r2.slugsTouched.sort()).toEqual(r1.slugsTouched.sort());
});
});
describe('redaction before write', () => {
test('planted secret never reaches the page; redaction counted', async () => {
const p = join(tmp, 'secret-session.jsonl');
writeFileSync(
p,
[
JSON.stringify({ type: 'session', version: 3, id: 'secret-session-01', timestamp: '2026-08-09T10:00:00.000Z' }),
JSON.stringify({
type: 'message',
id: 'm-1',
timestamp: '2026-08-09T10:00:01.000Z',
message: {
role: 'user',
timestamp: '2026-08-09T10:00:01.000Z',
content: [{ type: 'text', text: `the deploy key is ${PLANTED_KEY} keep it safe` }],
},
}),
].join('\n') + '\n',
);
const r = await runTranscriptsIngest(engine, baseOpts([p]));
expect(r.sessionsImported).toBe(1);
expect(r.redactions).toBeGreaterThanOrEqual(1);
const page = await engine.getPage(r.slugsTouched[0], { sourceId: 'default' });
expect(page).not.toBeNull();
expect(page!.compiled_truth).not.toContain(PLANTED_KEY);
expect(page!.compiled_truth).toContain('<REDACTED:');
});
});
/**
* Wave B: a new session is one page under the page ceiling, so a session splits only past it. These tests lower
* `pages.max_bytes` for the first import to get a split session; its parts pin that part size afterwards.
*/
async function ingestUnderCeiling(maxBytes: number, paths: string[]) {
await engine.setConfig('pages.max_bytes', String(maxBytes));
try {
return await runTranscriptsIngest(engine, baseOpts(paths));
} finally {
await engine.unsetConfig('pages.max_bytes');
}
}
describe('part splitting + dangerous transitions', () => {
test('a big new session imports as one embeddable page; a session past the ceiling splits into real pages', async () => {
const p = writeBigAgentSession(tmp, 'bigsession-0001', 150);
const r = await runTranscriptsIngest(engine, baseOpts([p]));
expect(r.sessionsImported).toBe(1);
expect(r.pages.imported).toBe(1);
expect(r.cleanScan).toBe(true); // pristine synthetic file: clean scan holds
const base = buildTranscriptSlug('openclaw', '2026-08-10T08:00:00.000Z', {
sessionId: 'bigsession-0001',
});
const one = await engine.getPage(base, { sourceId: 'default' });
expect(one).not.toBeNull();
expect(await engine.getPage(`${base}-p2`, { sourceId: 'default' })).toBeNull();
expect((one!.frontmatter as Record<string, any>).embed_skip).toBeUndefined();
expect((one!.frontmatter as Record<string, any>).transcript_import).toMatchObject({ part: 1, of: 1 });
const q = writeBigAgentSession(tmp, 'bigsession-0002', 150);
const split = await ingestUnderCeiling(100_000, [q]);
expect(split.pages.imported).toBeGreaterThan(1);
const splitBase = buildTranscriptSlug('openclaw', '2026-08-10T08:00:00.000Z', {
sessionId: 'bigsession-0002',
});
const p1 = await engine.getPage(splitBase, { sourceId: 'default' });
const p2 = await engine.getPage(`${splitBase}-p2`, { sourceId: 'default' });
expect(p1).not.toBeNull();
expect(p2).not.toBeNull();
// Split pages must stay embeddable: no embed_skip marker on any part.
for (const page of [p1!, p2!]) {
const fm = page.frontmatter as Record<string, any>;
expect(fm.embed_skip).toBeUndefined();
expect(fm.transcript_import.of).toBeUndefined();
expect(fm.transcript_import.part_bytes).toBe(90_000);
}
// Unique per-part identity (a shared id would dedup-skip parts 2..N).
expect((p1!.frontmatter as any).id).not.toBe((p2!.frontmatter as any).id);
});
test('split → shrink deletes stale higher parts (reconciliation)', async () => {
const big = writeBigAgentSession(tmp, 'shrinksession-01', 150);
const r1 = await ingestUnderCeiling(100_000, [big]);
const parts = r1.pages.imported;
expect(parts).toBeGreaterThan(1);
// Same session id, now tiny: re-render to ONE part.
const small = writeBigAgentSession(join(tmp), 'shrinksession-01', 2);
const r2 = await runTranscriptsIngest(engine, baseOpts([small]));
expect(r2.sessionsImported).toBe(1);
expect(r2.partsDeleted).toBe(parts - 1);
const base = buildTranscriptSlug('openclaw', '2026-08-10T08:00:00.000Z', {
sessionId: 'shrinksession-01',
});
expect(await engine.getPage(base, { sourceId: 'default' })).not.toBeNull();
expect(await engine.getPage(`${base}-p2`, { sourceId: 'default' })).toBeNull();
});
test('reconciliation heals crash holes (deleted -p2, surviving -p3)', async () => {
const big = writeBigAgentSession(tmp, 'holesession-0001', 300);
const r1 = await ingestUnderCeiling(100_000, [big]);
expect(r1.pages.imported).toBeGreaterThan(2); // need at least p3 for the hole
const base = buildTranscriptSlug('openclaw', '2026-08-10T08:00:00.000Z', {
sessionId: 'holesession-0001',
});
// Simulate a crash mid-reconciliation on a prior shrink: -p2 already
// deleted, higher parts survive.
await engine.deletePage(`${base}-p2`, { sourceId: 'default' });
const small = writeBigAgentSession(join(tmp), 'holesession-0001', 2);
const r2 = await runTranscriptsIngest(engine, baseOpts([small]));
expect(r2.sessionsImported).toBe(1);
// SQL enumeration walks past the -p2 hole and removes every survivor.
expect(await engine.getPage(`${base}-p3`, { sourceId: 'default' })).toBeNull();
});
});
describe('since/limit clean-scan semantics', () => {
test('sinceIso filters old sessions; limit counts NEW WORK; truncation breaks cleanScan', async () => {
// Two pristine synthetic sessions (no malformed lines → clean scans).
const a = writeBigAgentSession(tmp, 'sincesession-0001', 2);
const b = writeBigAgentSession(tmp, 'sincesession-0002', 2);
// Both are older than the since bound → filtered, clean scan holds.
const rSince = await runTranscriptsIngest(
engine,
baseOpts([a, b], { sinceIso: '2027-01-01T00:00:00.000Z' }),
);
expect(rSince.sessionsFiltered).toBe(2);
expect(rSince.sessionsImported).toBe(0);
expect(rSince.cleanScan).toBe(true);
expect(rSince.maxSessionTs > '2026-08-01').toBe(true);
// limit=1 over two files → truncated, NOT a clean scan (watermark frozen).
const rLimit = await runTranscriptsIngest(engine, baseOpts([a, b], { limit: 1 }));
expect(rLimit.sessionsImported).toBe(1);
expect(rLimit.cleanScan).toBe(false);
// Kill/rerun convergence WITH the same limit: hash-skipped re-scans are
// FREE (they don't burn the limit), so run 2 reaches the second session
// instead of looping over the imported prefix forever.
const rLimit2 = await runTranscriptsIngest(engine, baseOpts([a, b], { limit: 1 }));
expect(rLimit2.sessionsImported).toBe(2); // 1 hash-skip + 1 new import
const rFull = await runTranscriptsIngest(engine, baseOpts([a, b]));
expect(rFull.pages.imported).toBe(0);
expect(rFull.pages.skipped).toBe(2);
expect(rFull.cleanScan).toBe(true);
});
});
describe('error taxonomy', () => {
test('unknown-format file is a per-file error; the run continues', async () => {
const junk = join(tmp, 'junk.jsonl');
writeFileSync(junk, '{"unrelated":true}\n');
const r = await runTranscriptsIngest(engine, baseOpts([junk, CODEX_FIXTURE]));
expect(r.erroredFiles).toBe(1);
expect(r.sessionsImported).toBe(1);
expect(r.cleanScan).toBe(false);
});
test('zero-session file raises the drift signal', async () => {
const empty = join(tmp, 'empty.jsonl');
writeFileSync(
empty,
JSON.stringify({ type: 'session', version: 3, id: 'empty-session-1', timestamp: '2026-08-09T10:00:00.000Z' }) + '\n',
);
const r = await runTranscriptsIngest(engine, baseOpts([empty], { format: 'openclaw' }));
expect(r.driftFiles).toBe(1);
expect(r.sessionsImported).toBe(0);
});
});
describe('all seven formats travel the FULL pipeline (parse → redact → render → import)', () => {
test('claude-code: the shipped fixture imports as a page with placeholders and real timestamps', async () => {
const r = await runTranscriptsIngest(engine, baseOpts([CLAUDE_CODE_FIXTURE]));
expect(r.sessionsImported).toBe(1);
expect(r.pages.imported).toBe(1);
const slug = buildTranscriptSlug('claude-code', '2026-08-01T10:00:00.000Z', {
sessionId: 'fixture-session-1',
});
const page = await engine.getPage(slug, { sourceId: 'default' });
expect(page).not.toBeNull();
expect(page!.type).toBe('conversation');
const fm = page!.frontmatter as Record<string, any>;
expect(fm.transcript_import.harness).toBe('claude-code');
expect(fm.date).toBe('2026-08-01');
// Text turns land; tool traffic appears only as placeholders; the
// anchor lines carry the fixture's REAL timestamps.
expect(page!.compiled_truth).toContain("widget-co's seed round");
expect(page!.compiled_truth).toContain('[tool: search_brain]');
expect(page!.compiled_truth).toContain('(2026-08-01 10:00 AM)');
});
test('hermes: ONE store file yields MANY pages (multi-session ingest path)', async () => {
const dbPath = buildHermesFixture(tmp);
const r = await runTranscriptsIngest(engine, baseOpts([dbPath]));
// 3 sessions in the store; the tool-only one never yields → 2 imported.
expect(r.sessionsImported).toBe(2);
expect(r.pages.imported).toBe(2);
expect(r.cleanScan).toBe(true);
const s1 = buildTranscriptSlug('hermes', '2026-08-05T08:00:00.000Z', {
sessionId: 'hermes-fixture-1',
});
const s2 = buildTranscriptSlug('hermes', '2026-08-06T08:00:00.000Z', {
sessionId: 'hermes-fixture-2',
});
const p1 = await engine.getPage(s1, { sourceId: 'default' });
const p2 = await engine.getPage(s2, { sourceId: 'default' });
expect(p1).not.toBeNull();
expect(p2).not.toBeNull();
// Title is promoted to the page COLUMN at import (not kept in frontmatter).
expect(p1!.title).toContain('widget planning');
// Session 2's JSON block-array contents unwrapped to text in the page.
expect(p2!.compiled_truth).toContain('acme-seed closes at the end of the month.');
// Session metadata rode raw_data for BOTH sessions of the one file.
const raw1 = await engine.getRawData(s1, 'transcript:hermes', { sourceId: 'default' });
const raw2 = await engine.getRawData(s2, 'transcript:hermes', { sourceId: 'default' });
expect(raw1.length).toBe(1);
expect(raw2.length).toBe(1);
// limit interplay on a multi-session FILE: limit=1 imports one session,
// truncates cleanly, and the follow-up run converges.
await resetPgliteState(engine);
const rLimit = await runTranscriptsIngest(engine, baseOpts([dbPath], { limit: 1 }));
expect(rLimit.sessionsImported).toBe(1);
expect(rLimit.cleanScan).toBe(false);
const rRest = await runTranscriptsIngest(engine, baseOpts([dbPath]));
expect(rRest.pages.imported + rRest.pages.skipped).toBe(2);
});
test('grok: chat_history.jsonl imports text turns; tools/system/synthetic never land', async () => {
const r = await runTranscriptsIngest(engine, baseOpts([GROK_FIXTURE]));
expect(r.sessionsImported).toBe(1);
expect(r.pages.imported).toBe(1);
const slug = buildTranscriptSlug('grok', '2026-08-08T11:00:00.000Z', {
sessionId: 'grok-fixture-session-1',
});
const page = await engine.getPage(slug, { sourceId: 'default' });
expect(page).not.toBeNull();
expect(page!.type).toBe('conversation');
const fm = page!.frontmatter as Record<string, any>;
expect(fm.transcript_import.harness).toBe('grok');
expect(fm.date).toBe('2026-08-08');
expect(page!.compiled_truth).toContain('Which fund led the widget-co seed');
expect(page!.compiled_truth).toContain('fund-a led the widget-co seed');
expect(page!.compiled_truth).not.toContain('SYSTEM-ONLY-TEXT');
expect(page!.compiled_truth).not.toContain('TOOL-OUTPUT-ONLY-TEXT');
expect(page!.compiled_truth).not.toContain('SYNTHETIC-ONLY-TEXT');
const raw = await engine.getRawData(slug, 'transcript:grok', { sourceId: 'default' });
expect(raw.length).toBe(1);
});
test('chatgpt export: one file → per-thread pages under conversations/chatgpt/ with title slugs', async () => {
const r = await runTranscriptsIngest(engine, baseOpts([CHATGPT_FIXTURE]));
// Conversation 3 is system-only → skipped by the adapter.
expect(r.sessionsImported).toBe(2);
expect(r.pages.imported).toBe(2);
const slug = buildTranscriptSlug('chatgpt', new Date(1786080000 * 1000).toISOString(), {
sessionId: 'cgpt-conv-0001',
title: 'Widget launch naming',
});
expect(slug).toContain('conversations/chatgpt/');
expect(slug).toContain('widget-launch-naming');
const page = await engine.getPage(slug, { sourceId: 'default' });
expect(page).not.toBeNull();
expect(page!.title).toBe('Widget launch naming');
// Canonical path only — the abandoned branch never lands in the page.
expect(page!.compiled_truth).toContain('Call it LaunchPanel.');
expect(page!.compiled_truth).not.toContain('BRANCH-A-ONLY-TEXT');
});
test('claude.ai export: one file → pages under conversations/claude/ with title slugs', async () => {
const r = await runTranscriptsIngest(engine, baseOpts([CLAUDE_EXPORT_FIXTURE]));
expect(r.sessionsImported).toBe(1);
expect(r.pages.imported).toBe(1);
const slug = buildTranscriptSlug('claude-export', '2026-08-07T12:00:00.000Z', {
sessionId: 'claude-conv-0001',
title: 'Deal memo review',
});
expect(slug).toContain('conversations/claude/');
expect(slug).toContain('deal-memo-review');
const page = await engine.getPage(slug, { sourceId: 'default' });
expect(page).not.toBeNull();
expect(page!.compiled_truth).toContain('fund-a term sheet date');
const fm = page!.frontmatter as Record<string, any>;
expect(fm.transcript_import.harness).toBe('claude-export');
});
});
describe('raw metadata redaction [security: raw rides the REDACTED copy]', () => {
test('secrets in session metadata never reach raw_data', async () => {
const p = join(tmp, 'meta-secret.jsonl');
writeFileSync(
p,
[
JSON.stringify({
type: 'session',
version: 3,
id: 'meta-secret-01',
timestamp: '2026-08-09T10:00:00.000Z',
cwd: `/home/alice/${PLANTED_KEY}-project`,
}),
JSON.stringify({
type: 'message',
id: 'm-1',
timestamp: '2026-08-09T10:00:01.000Z',
message: {
role: 'user',
timestamp: '2026-08-09T10:00:01.000Z',
content: [{ type: 'text', text: 'plain question' }],
},
}),
].join('\n') + '\n',
);
const r = await runTranscriptsIngest(engine, baseOpts([p]));
expect(r.sessionsImported).toBe(1);
const raw = await engine.getRawData(r.slugsTouched[0], undefined, { sourceId: 'default' });
expect(raw.length).toBeGreaterThan(0);
const stored = JSON.stringify(raw[0].data);
expect(stored).not.toContain(PLANTED_KEY);
expect(stored).toContain('<REDACTED:');
});
});
describe('embed-OFF default', () => {
test('imported pages carry zero embedded chunks unless embed is opted in', async () => {
const r = await runTranscriptsIngest(engine, baseOpts([CODEX_FIXTURE]));
expect(r.pages.imported).toBe(1);
const rows = await engine.executeRaw<{ n: number }>(
`SELECT COUNT(*)::int AS n FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
WHERE p.slug = $1 AND cc.embedding IS NOT NULL`,
[r.slugsTouched[0]],
);
expect(Number(rows[0].n)).toBe(0);
});
});
describe('discovery + status [injected roots, never the real home]', () => {
test('discovery filters symlinks/checkpoints; status gap math catches late arrivals', async () => {
// Fake harness layout: an openclaw agents tree with one real session,
// one checkpoint sibling, one symlink; plus a codex sessions tree.
const openclawRoot = join(tmp, 'agents');
const codexRoot = join(tmp, 'sessions');
mkdirSync(join(openclawRoot, 'main', 'sessions'), { recursive: true });
mkdirSync(codexRoot, { recursive: true });
const realSession = join(openclawRoot, 'main', 'sessions', 'agent-fixture-session-1.jsonl');
copyFileSync(AGENT_FIXTURE, realSession);
copyFileSync(
AGENT_FIXTURE,
join(openclawRoot, 'main', 'sessions', 'agent-fixture-session-1.checkpoint.aaaa-bbbb.jsonl'),
);
symlinkSync(realSession, join(openclawRoot, 'main', 'sessions', 'link.jsonl'));
copyFileSync(CODEX_FIXTURE, join(codexRoot, 'rollout-codex-fixture-session-1.jsonl'));
const roots: HarnessRoot[] = [
{ format: 'openclaw', root: openclawRoot, extension: '.jsonl' },
{ format: 'codex', root: codexRoot, extension: '.jsonl' },
];
const discovered = discoverTranscriptFiles(roots);
// Checkpoint + symlink excluded: one file per harness.
expect(discovered.map((d) => d.format).sort()).toEqual(['codex', 'openclaw']);
// Import ONLY the openclaw session; codex stays a gap (late arrival).
const r = await runTranscriptsIngest(engine, baseOpts([realSession]));
expect(r.sessionsImported).toBe(1);
const rows = buildStatusRows(discovered, await indexImportedSessions(engine, 'default'), roots);
const oc = rows.find((x) => x.format === 'openclaw')!;
const cx = rows.find((x) => x.format === 'codex')!;
expect(oc.found).toBe(1);
expect(oc.importedSessions).toBe(1);
expect(oc.gapFiles).toBe(0);
expect(cx.found).toBe(1);
expect(cx.importedSessions).toBe(0);
expect(cx.gapFiles).toBe(1); // the watermark-blind late arrival, caught here
});
});
describe('facts kill-switch pre-check', () => {
test('facts.extraction_enabled=false skips with a notice result, never a throw', async () => {
await engine.setConfig('facts.extraction_enabled', 'false');
try {
const r = await runIngestFacts(engine, {
sourceId: 'default',
slugs: ['conversations/sessions/whatever'],
quiet: true,
});
expect(r.skippedDisabled).toBe(true);
expect(r.pages).toBe(0);
} finally {
await engine.unsetConfig('facts.extraction_enabled');
}
});
});
describe('putRawData zero-row parity (PGLite)', () => {
test('missing page throws instead of silently no-opping', async () => {
await expect(
engine.putRawData('conversations/sessions/never-imported', 'transcript:codex', { a: 1 }, { sourceId: 'default' }),
).rejects.toThrow(/not found/);
await expect(
engine.putRawData('conversations/sessions/never-imported-2', 'transcript:codex', { a: 1 }),
).rejects.toThrow(/not found/);
});
});
// ── Format-based redaction lands in the STORED page ─────────────────────────
//
// Synthetic, runtime-joined values (>= 2 fragments each) so no committed line
// carries a credential-shaped literal. Read back through the engine (never
// grep the PGLite data dir — its paged format hides plaintext from grep).
const SEEDED_JWT = [
'eyJhbGciOiJIUzI1NiJ9',
'eyJyb2xlIjoic2VydmljZV9yb2xlIiwiaWF0IjoxNzAwMDAwMDAwfQ',
'c2lnbmF0dXJlLXBsYWNlaG9sZGVyLTAwMDA',
].join('.');
const SEEDED_SID = ['AC', '0123456789abcdef', '0123456789abcdef'].join('');
const SEEDED_GOOGLE = ['AIza', 'SyD1-Fake_Example0123456789abcdefGH'].join('');
const SEEDED_STRIPE = ['sk_live_', '4eC39HqLyjWDarjtT1zdp7dc'].join('');
const SEEDED_DB_URL = ['postgres://', 'dbuser', ':', 'p4ssw0rd', '@db.internal:5432/app'].join('');
const SEEDED_ENTROPIC = ['aB3xK9mQ', '2pR7sT1vW4yZ8bC5'].join('');
const SEEDED_OPAQUE = ['opaque', 'Token0123456789abcdefXYZ'].join('');
function claudeCodeLine(type: 'user' | 'assistant', uuid: string, ts: string, content: string): string {
return JSON.stringify({
type,
message: { role: type, content },
uuid,
timestamp: ts,
cwd: '/tmp/poc',
sessionId: 'poc-0001',
version: '2.1.0',
gitBranch: 'main',
});
}
describe('format-based redaction before write (unprefixed credential shapes)', () => {
test('claude-code session: JWT / account SID / connection string / cloud keys land as placeholders only — and a bare echo in ANOTHER message goes with them', async () => {
const p = join(tmp, 'poc-session.jsonl');
writeFileSync(
p,
[
claudeCodeLine('user', 'seed-0000', '2026-08-25T23:00:00.000Z', `deploy is failing. service role key ${SEEDED_JWT} and project ref zfakerefzfakeref0000`),
// The opaque token appears BARE here, two messages before the header
// that claims it; the entropic value appears bare AFTER its claim.
claudeCodeLine('assistant', 'seed-0001', '2026-08-25T23:00:01.000Z', `twilio sid ${SEEDED_SID} and maps key ${SEEDED_GOOGLE} need rotating; the token ${SEEDED_OPAQUE} expired too`),
claudeCodeLine('user', 'seed-0002', '2026-08-25T23:00:02.000Z', `stripe ${SEEDED_STRIPE} plus DATABASE_URL=${SEEDED_DB_URL} and SMTP_TOKEN=${SEEDED_ENTROPIC}`),
claudeCodeLine('assistant', 'seed-0003', '2026-08-25T23:00:03.000Z', `retried with the header Authorization: Bearer ${SEEDED_OPAQUE} and pasted ${SEEDED_ENTROPIC} into the env`),
].join('\n') + '\n',
);
const r = await runTranscriptsIngest(engine, baseOpts([p]));
expect(r.sessionsImported).toBe(1);
expect(r.pages.imported).toBe(1);
// Seven CLAIMS; the two bare echoes add no count.
expect(r.redactions).toBeGreaterThanOrEqual(7);
const slug = r.slugsTouched[0];
const page = await engine.getPage(slug, { sourceId: 'default' });
expect(page).not.toBeNull();
const body = page!.compiled_truth;
// Positive control: harmless prose from the same page is present.
expect(body).toContain('deploy is failing');
for (const v of [SEEDED_JWT, SEEDED_SID, SEEDED_GOOGLE, SEEDED_STRIPE, SEEDED_ENTROPIC, SEEDED_OPAQUE, 'p4ssw0rd']) {
expect(body).not.toContain(v);
}
for (const tag of [
'<REDACTED:jwt>', '<REDACTED:twilio>', '<REDACTED:google_api_key>', '<REDACTED:stripe>',
'<REDACTED:db_url_credentials>', '<REDACTED:high_entropy_assignment>', '<REDACTED:bearer>',
]) {
expect(body).toContain(tag);
}
// The session-wide echo dictionary: the token's bare mention in an
// EARLIER message and the entropic value's in a LATER one are scrubbed.
expect(body).toContain('the token <REDACTED:bearer> expired too');
expect(body).toContain('pasted <REDACTED:high_entropy_assignment> into the env');
// A project ref is an identifier, not a credential — deliberately kept.
expect(body).toContain('zfakerefzfakeref0000');
// Whatever session metadata rode along carries no seeded value either.
const raw = await engine.getRawData(slug, undefined, { sourceId: 'default' });
const stored = JSON.stringify(raw);
for (const v of [SEEDED_JWT, SEEDED_SID, SEEDED_GOOGLE, SEEDED_STRIPE, 'p4ssw0rd']) {
expect(stored).not.toContain(v);
}
});
});
describe('AI-provider keys (#7959) are redacted before write', () => {
test('a bare key in a message lands as its own placeholder; neither body nor raw metadata keeps it', async () => {
const keys = PROVIDER_KEY_SHAPES.map((s) => ({ pattern: s.pattern, key: s.make() }));
const p = join(tmp, 'provider-keys-session.jsonl');
writeFileSync(
p,
keys.map(({ pattern, key }, i) =>
claudeCodeLine(i % 2 ? 'assistant' : 'user', `pk-000${i}`, `2026-08-26T10:00:0${i}.000Z`, `the ${pattern} key is ${key} for now`),
).join('\n') + '\n',
);
const r = await runTranscriptsIngest(engine, baseOpts([p]));
expect(r.pages.imported).toBe(1);
expect(r.redactions).toBeGreaterThanOrEqual(keys.length);
const slug = r.slugsTouched[0];
const body = (await engine.getPage(slug, { sourceId: 'default' }))!.compiled_truth;
const stored = JSON.stringify(await engine.getRawData(slug, undefined, { sourceId: 'default' }));
for (const { pattern, key } of keys) {
expect(body).toContain(`the ${pattern} key is <REDACTED:${pattern}> for now`);
expect(body).not.toContain(key);
expect(stored).not.toContain(key);
}
});
});
/** Count calls to an engine method for one block; restores the original. */
async function spyEngine<K extends 'getRawData' | 'putRawData'>(
method: K,
run: () => Promise<void>,
): Promise<unknown[][]> {
const calls: unknown[][] = [];
const original = (engine as any)[method].bind(engine);
(engine as any)[method] = (...args: unknown[]) => { calls.push(args); return original(...args); };
try { await run(); } finally { delete (engine as any)[method]; }
return calls;
}
function writePlainSession(path: string): void {
writeFileSync(
path,
[
JSON.stringify({ type: 'session', version: 3, id: 'plain-02', timestamp: '2026-08-09T11:00:00.000Z', cwd: '/tmp/plain' }),
JSON.stringify({
type: 'message',
id: 'm-1',
timestamp: '2026-08-09T11:00:01.000Z',
message: { role: 'user', timestamp: '2026-08-09T11:00:01.000Z', content: [{ type: 'text', text: 'plain question two' }] },
}),
].join('\n') + '\n',
);
}
describe('re-ingest over a soft-deleted base page', () => {
test('resurrects the page (hash check reads ACTIVE rows only) — the compare probe never runs, raw is rewritten', async () => {
const p = join(tmp, 'plain-session-2.jsonl');
writePlainSession(p);
const r1 = await runTranscriptsIngest(engine, baseOpts([p]));
expect(r1.pages.imported).toBe(1);
const slug = r1.slugsTouched[0];
expect(await engine.softDeletePage(slug, { sourceId: 'default' })).not.toBeNull();
expect(await engine.getPage(slug, { sourceId: 'default' })).toBeNull();
let r2: Awaited<ReturnType<typeof runTranscriptsIngest>> | undefined;
let putCalls: unknown[][] = [];
const probeCalls = await spyEngine('getRawData', async () => {
putCalls = await spyEngine('putRawData', async () => {
r2 = await runTranscriptsIngest(engine, baseOpts([p]));
});
});
// No abort; the tombstoned base page reads as missing to the import
// hash check, so it is IMPORTED again (not skipped) and comes back alive.
expect(r2!.sessionsImported).toBe(1);
expect(r2!.sessionsErrored).toBe(0);
expect(r2!.cleanScan).toBe(true);
expect(r2!.pages.imported).toBe(1);
expect(r2!.pages.skipped).toBe(0);
const alive = await engine.getPage(slug, { sourceId: 'default' });
expect(alive).not.toBeNull();
expect(alive!.deleted_at).toBeNull();
// Because the session was NOT all-skipped, the compare-before-write
// probe is not reached; the raw row is written fresh over the
// resurrected page. (The probe reads ACTIVE rows only — it never needs
// to see through a tombstone, because a tombstoned base page never
// reaches the all-skipped branch: this test is that proof.)
expect(probeCalls).toEqual([]);
expect(putCalls.length).toBe(1);
expect((await engine.getRawData(slug, undefined, { sourceId: 'default' })).length).toBe(1);
});
test('the compare probe fires only on an all-skipped (alive) re-run, and a content-equal row is not rewritten', async () => {
const p = join(tmp, 'plain-session-3.jsonl');
writePlainSession(p);
const r1 = await runTranscriptsIngest(engine, baseOpts([p]));
const slug = r1.slugsTouched[0];
let putCalls: unknown[][] = [];
const probeCalls = await spyEngine('getRawData', async () => {
putCalls = await spyEngine('putRawData', async () => {
const r2 = await runTranscriptsIngest(engine, baseOpts([p]));
expect(r2.pages.skipped).toBe(1);
expect(r2.pages.imported).toBe(0);
});
});
expect(probeCalls.length).toBe(1);
expect(probeCalls[0][0]).toBe(slug);
expect(probeCalls[0][1]).toBe('transcript:openclaw');
expect(probeCalls[0][2]).toEqual({ sourceId: 'default' }); // active rows only — no includeDeleted
expect(putCalls).toEqual([]);
});
});
describe('raw_data follows the page soft-delete', () => {
test('after softDeletePage the session metadata is invisible; includeDeleted:true still reads it; restore brings it back', async () => {
const p = join(tmp, 'plain-session.jsonl');
writeFileSync(
p,
[
JSON.stringify({ type: 'session', version: 3, id: 'plain-01', timestamp: '2026-08-09T10:00:00.000Z', cwd: '/tmp/plain' }),
JSON.stringify({
type: 'message',
id: 'm-1',
timestamp: '2026-08-09T10:00:01.000Z',
message: { role: 'user', timestamp: '2026-08-09T10:00:01.000Z', content: [{ type: 'text', text: 'plain question' }] },
}),
].join('\n') + '\n',
);
const r = await runTranscriptsIngest(engine, baseOpts([p]));
expect(r.sessionsImported).toBe(1);
const slug = r.slugsTouched[0];
expect((await engine.getRawData(slug, undefined, { sourceId: 'default' })).length).toBeGreaterThan(0);
expect(await engine.softDeletePage(slug, { sourceId: 'default' })).not.toBeNull();
expect(await engine.getRawData(slug, undefined, { sourceId: 'default' })).toEqual([]);
expect(await engine.getRawData(slug)).toEqual([]);
expect((await engine.getRawData(slug, undefined, { sourceId: 'default', includeDeleted: true })).length).toBeGreaterThan(0);
expect(await engine.restorePage(slug, { sourceId: 'default' })).toBe(true);
expect((await engine.getRawData(slug, undefined, { sourceId: 'default' })).length).toBeGreaterThan(0);
});
});