Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

11.4 KB
Raw Blame
import { createReadStream, existsSync } from 'node:fs';
import { readdir, readFile, realpath } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import { createInterface } from 'node:readline';
import type {
  BrowserUnavailableReason,
  ThreadEvent,
  ThreadInfo,
} from '@inkeep/open-knowledge-core/acp/thread-protocol';
import {
  tracedAppendFile,
  tracedMkdir,
  tracedRename,
  tracedRm,
  tracedWriteFile,
} from '../fs-traced.ts';
import type { PinoLogger } from '../logger.ts';

const THREADS_SUBDIR = 'threads';
const META_VERSION = 1;
const READ_CHUNK_SIZE = 512;

export function acpThreadsDir(baseDir: string): string {
  return join(baseDir, THREADS_SUBDIR);
}

export function acpThreadStoreRoots(globalDir: string | null, localDir: string | null): string[] {
  return [globalDir, localDir].filter((dir) => dir !== null);
}

const MINTED_THREAD_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;

export function isMintedThreadId(threadId: string): boolean {
  return MINTED_THREAD_ID.test(threadId);
}

export interface PersistedThreadMeta {
  version: typeof META_VERSION;
  info: ThreadInfo;
  sessionId: string | null;
  cwd: string;
  agentRef: { source: 'registry' | 'custom'; id: string };
  docName?: string;
  contextWindow?: number | null;
  browserNotice?: BrowserUnavailableReason | null;
}

export interface ResolvedEventLog {
  count: number;
  midTurn: boolean;
}

export interface ThreadPersistenceStoreOptions {
  primaryDir: string;
  legacyDir?: string | null;
  cwd?: string | null;
  log: PinoLogger;
}

export class ThreadPersistenceStore {
  private readonly primaryThreadsDir: string;
  private readonly legacyThreadsDir: string | null;
  private readonly cwd: string | null;
  private readonly log: PinoLogger;
  private writeThreadsDir: string;
  private readonly homeByThread = new Map<string, string>();
  private readonly writeQueues = new Map<string, Promise<void>>();
  private readonly appendBroken = new Set<string>();

  constructor(opts: ThreadPersistenceStoreOptions) {
    this.primaryThreadsDir = acpThreadsDir(opts.primaryDir);
    this.legacyThreadsDir = opts.legacyDir != null ? acpThreadsDir(opts.legacyDir) : null;
    this.cwd = opts.cwd ?? null;
    this.log = opts.log;
    this.writeThreadsDir = this.primaryThreadsDir;
  }

  private threadsDirFor(threadId: string): string {
    return this.homeByThread.get(threadId) ?? this.writeThreadsDir;
  }

  private pinHome(threadId: string): void {
    if (!this.homeByThread.has(threadId)) this.homeByThread.set(threadId, this.writeThreadsDir);
  }

  private enqueue(threadId: string, task: () => Promise<void>): void {
    const prev = this.writeQueues.get(threadId) ?? Promise.resolve();
    this.writeQueues.set(threadId, prev.then(task));
  }

  async init(): Promise<void> {
    try {
      await tracedMkdir(this.primaryThreadsDir, { recursive: true });
    } catch (err) {
      this.log.warn(
        { err, dir: this.primaryThreadsDir },
        '[acp-persist] primary threads dir unavailable',
      );
      if (this.legacyThreadsDir !== null) {
        this.writeThreadsDir = this.legacyThreadsDir;
        try {
          await tracedMkdir(this.legacyThreadsDir, { recursive: true });
        } catch (legacyErr) {
          this.log.error(
            { err: legacyErr, dir: this.legacyThreadsDir },
            '[acp-persist] legacy threads dir also unavailable; threads are memory-only',
          );
        }
      }
    }
  }

  eventsPath(threadId: string): string {
    return join(this.threadsDirFor(threadId), `${threadId}.ndjson`);
  }

  metaPath(threadId: string): string {
    return join(this.threadsDirFor(threadId), `${threadId}.meta.json`);
  }

  appendEvents(threadId: string, events: readonly ThreadEvent[]): void {
    if (events.length === 0 || this.appendBroken.has(threadId)) return;
    this.pinHome(threadId);
    const lines = `${events.map((e) => JSON.stringify(e)).join('\n')}\n`;
    this.enqueue(threadId, async () => {
      if (this.appendBroken.has(threadId)) return;
      try {
        await tracedAppendFile(this.eventsPath(threadId), lines);
      } catch (err) {
        this.appendBroken.add(threadId);
        this.log.error(
          { err, threadId },
          '[acp-persist] event append failed; thread continues memory-only',
        );
      }
    });
  }

  queueMetaWrite(threadId: string, meta: PersistedThreadMeta): void {
    this.pinHome(threadId);
    const body = `${JSON.stringify(meta, null, 1)}\n`;
    this.enqueue(threadId, async () => {
      const path = this.metaPath(threadId);
      const tmp = `${path}.tmp`;
      try {
        await tracedWriteFile(tmp, body);
        await tracedRename(tmp, path);
      } catch (err) {
        this.log.warn({ err, threadId }, '[acp-persist] meta write failed');
        try {
          await tracedRm(tmp, { force: true });
        } catch {}
      }
    });
  }

  whenIdle(threadId: string): Promise<void> {
    return this.writeQueues.get(threadId) ?? Promise.resolve();
  }

  async scan(): Promise<PersistedThreadMeta[]> {
    const metas: PersistedThreadMeta[] = [];
    const seen = new Set<string>();
    const canonCwd = this.cwd !== null ? await canonicalPath(this.cwd) : null;
    await this.scanDir(this.primaryThreadsDir, canonCwd, metas, seen);
    if (this.legacyThreadsDir !== null && this.legacyThreadsDir !== this.primaryThreadsDir) {
      await this.scanDir(this.legacyThreadsDir, null, metas, seen);
    }
    return metas;
  }

  private async scanDir(
    dir: string,
    canonCwd: string | null,
    out: PersistedThreadMeta[],
    seen: Set<string>,
  ): Promise<void> {
    let names: string[];
    try {
      names = await readdir(dir);
    } catch (err) {
      if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
        this.log.warn({ err, dir }, '[acp-persist] could not read a threads dir');
      }
      return;
    }
    for (const name of names) {
      if (!name.endsWith('.meta.json')) continue;
      const path = join(dir, name);
      let parsed: Partial<PersistedThreadMeta>;
      try {
        parsed = JSON.parse(await readFile(path, 'utf8')) as Partial<PersistedThreadMeta>;
      } catch (err) {
        this.log.warn({ err, path }, '[acp-persist] skipping unreadable thread meta');
        continue;
      }
      if (
        parsed.version !== META_VERSION ||
        typeof parsed.info !== 'object' ||
        parsed.info === null ||
        typeof parsed.info.threadId !== 'string' ||
        typeof parsed.cwd !== 'string' ||
        typeof parsed.agentRef !== 'object' ||
        parsed.agentRef === null
      ) {
        this.log.warn({ path }, '[acp-persist] skipping unreadable thread meta');
        continue;
      }
      if (!isMintedThreadId(parsed.info.threadId) || name !== `${parsed.info.threadId}.meta.json`) {
        this.log.warn(
          { path },
          '[acp-persist] skipping a thread meta whose id OpenKnowledge did not mint',
        );
        continue;
      }
      if (typeof parsed.sessionId !== 'string' && parsed.sessionId !== null) {
        this.log.warn(
          { path, invalidSessionId: parsed.sessionId },
          '[acp-persist] thread meta has no usable session id; keeping the transcript as unresumable',
        );
        parsed.sessionId = null;
      }
      const window = parsed.contextWindow;
      if (window !== undefined && window !== null) {
        if (typeof window !== 'number' || !Number.isFinite(window) || window <= 0) {
          this.log.warn(
            { path, invalidContextWindow: window },
            '[acp-persist] thread meta has an unusable context window; relaunching on the agent default',
          );
          parsed.contextWindow = null;
        }
      }
      const meta = parsed as PersistedThreadMeta;
      const threadId = meta.info.threadId;
      if (seen.has(threadId)) continue;
      if (canonCwd !== null && (await canonicalPath(meta.cwd)) !== canonCwd) continue;
      seen.add(threadId);
      if (!this.homeByThread.has(threadId)) this.homeByThread.set(threadId, dir);
      out.push(meta);
    }
  }

  async resolveEventLog(threadId: string): Promise<ResolvedEventLog> {
    const path = this.eventsPath(threadId);
    if (!existsSync(path)) return { count: 0, midTurn: false };
    let count = 0;
    let midTurn = false;
    await this.forEachCompleteLine(path, (line) => {
      count += 1;
      if (line.startsWith('{"kind":"turn_started"')) midTurn = true;
      else if (line.startsWith('{"kind":"turn_ended"')) midTurn = false;
      return true;
    });
    return { count, midTurn };
  }

  async readEvents(
    threadId: string,
    fromSeq: number,
    toSeqExclusive: number,
    onChunk: (chunkFromSeq: number, events: ThreadEvent[]) => void,
  ): Promise<void> {
    const path = this.eventsPath(threadId);
    if (fromSeq >= toSeqExclusive || !existsSync(path)) return;
    let seq = 0;
    let chunk: ThreadEvent[] = [];
    let chunkFrom = fromSeq;
    const flush = (): void => {
      if (chunk.length === 0) return;
      onChunk(chunkFrom, chunk);
      chunkFrom += chunk.length;
      chunk = [];
    };
    await this.forEachCompleteLine(path, (line) => {
      if (seq >= toSeqExclusive) return false;
      if (seq >= fromSeq) {
        chunk.push(parseEventLine(line));
        if (chunk.length >= READ_CHUNK_SIZE) flush();
      }
      seq += 1;
      return true;
    });
    flush();
  }

  async delete(threadId: string): Promise<void> {
    this.appendBroken.delete(threadId);
    this.writeQueues.delete(threadId);
    const paths = [this.eventsPath(threadId), this.metaPath(threadId)];
    const results = await Promise.allSettled(paths.map((p) => tracedRm(p, { force: true })));
    this.homeByThread.delete(threadId);
    for (const [i, result] of results.entries()) {
      if (result.status === 'rejected') {
        this.log.warn(
          { err: result.reason, threadId, path: paths[i] },
          '[acp-persist] thread file removal failed',
        );
      }
    }
  }

  private async forEachCompleteLine(
    path: string,
    onLine: (line: string) => boolean,
  ): Promise<void> {
    const stream = createReadStream(path, { encoding: 'utf8' });
    let streamError: Error | null = null;
    stream.on('error', (err) => {
      streamError = err;
    });
    const rl = createInterface({ input: stream, crlfDelay: Number.POSITIVE_INFINITY });
    let endedWithNewline = false;
    let pending: string | null = null;
    let stopped = false;
    try {
      for await (const line of rl) {
        if (pending !== null && !onLine(pending)) {
          stopped = true;
          break;
        }
        pending = line;
      }
      if (!stopped) {
        endedWithNewline = await fileEndsWithNewline(path);
        if (pending !== null && endedWithNewline) onLine(pending);
      }
    } finally {
      rl.close();
      stream.destroy();
    }
    if (streamError !== null) throw streamError;
  }
}

async function fileEndsWithNewline(path: string): Promise<boolean> {
  const { open } = await import('node:fs/promises');
  const handle = await open(path, 'r');
  try {
    const { size } = await handle.stat();
    if (size === 0) return false;
    const buf = Buffer.alloc(1);
    await handle.read(buf, 0, 1, size - 1);
    return buf[0] === 0x0a;
  } finally {
    await handle.close();
  }
}

function parseEventLine(line: string): ThreadEvent {
  try {
    return JSON.parse(line) as ThreadEvent;
  } catch {
    return { kind: 'agent_stderr', line: '[unreadable log entry]', ts: 0 };
  }
}

async function canonicalPath(p: string): Promise<string> {
  try {
    return await realpath(p);
  } catch {
    return resolve(p);
  }
}