Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

4.9 KB
Raw Blame
/**
 * Raw filesystem access for transcript scanning.
 *
 * Isolated here so the rest of the usage code stays on Effect's `FileSystem`.
 * The direct `node:fs` streaming is deliberate: a cold 30-day window is ~1.4 GB
 * across ~1,500 files, and `readline` over a read stream is roughly an order of
 * magnitude cheaper than materialising each file. The equivalent Effect stream
 * pipeline is idiomatic but not fast enough to sit behind a page load.
 *
 * @module usageTranscriptReader
 */
import * as NodeFS from "node:fs";
import * as NodeFSP from "node:fs/promises";
import * as NodePath from "node:path";
import * as NodeReadline from "node:readline";

import type { UsageProviderKind } from "@akeru/contracts";

import {
  initialCodexScanState,
  mightCarryUsage,
  parseClaudeLine,
  parseCodexLine,
  type UsageRecord,
} from "./usageTranscripts.ts";

export interface TranscriptFile {
  readonly path: string;
  readonly size: number;
  readonly mtimeMs: number;
}

/**
 * Lists `.jsonl` transcripts under `root` last modified at or after `sinceMs`.
 *
 * Errors on individual entries are swallowed: session files rotate and get
 * removed while the walk is in flight, and a partial listing is far better than
 * failing the page.
 */
const walks = new Map<string, Promise<readonly TranscriptFile[]>>();

const reads = new Map<string, Promise<readonly UsageRecord[] | null>>();

export function listTranscriptFiles(
  root: string,
  sinceMs: number,
): Promise<readonly TranscriptFile[]> {
  let walk = walks.get(root);

  if (!walk) {
    walk = walkTranscriptFiles(root).finally(() => walks.delete(root));
    walks.set(root, walk);
  }

  return walk.then((files) => files.filter((file) => file.mtimeMs >= sinceMs));
}

async function walkTranscriptFiles(root: string): Promise<readonly TranscriptFile[]> {
  const found: TranscriptFile[] = [];

  const walk = async (dir: string): Promise<void> => {
    let entries;

    try {
      entries = await NodeFSP.readdir(dir, { withFileTypes: true });
    } catch {
      return;
    }

    for (const entry of entries) {
      const child = NodePath.join(dir, entry.name);

      if (entry.isDirectory()) {
        await walk(child);
        continue;
      }

      if (!entry.name.endsWith(".jsonl")) continue;

      try {
        const stats = await NodeFSP.stat(child);
        found.push({ path: child, size: stats.size, mtimeMs: stats.mtimeMs });
      } catch {
        // Vanished between readdir and stat.
      }
    }
  };

  await walk(root);

  return found;
}

/**
 * Filesystem identity of a directory, as `device:inode`.
 *
 * Used to tell "two servers reading the same transcript directory" apart from
 * "two machines whose hostname and home path happen to match". Returns an empty
 * string when the directory cannot be stat'd.
 */
export async function readDirectoryVolumeId(path: string): Promise<string> {
  try {
    const stats = await NodeFSP.stat(path);

    return `${stats.dev}:${stats.ino}`;
  } catch {
    return "";
  }
}

/**
 * Streams one transcript and returns the usage records it contains, or `null`
 * when the file could not be read.
 *
 * The distinction matters to the caller's cache: a genuinely empty transcript
 * is a stable fact worth memoising, while a transient read failure memoised
 * under the same `(size, mtime)` key would silently drop that file's usage
 * until the file next changes.
 *
 * Codex carries the active model on `turn_context` lines that hold no usage of
 * their own, so those still have to pass through the reducer to keep model
 * attribution correct.
 */
export function readTranscriptRecords(
  filePath: string,
  provider: UsageProviderKind,
  version?: Pick<TranscriptFile, "size" | "mtimeMs">,
): Promise<readonly UsageRecord[] | null> {
  const key = JSON.stringify([filePath, provider, version?.size, version?.mtimeMs]);
  let read = reads.get(key);

  if (!read) {
    read = scanTranscriptRecords(filePath, provider).finally(() => reads.delete(key));
    reads.set(key, read);
  }

  return read;
}

async function scanTranscriptRecords(
  filePath: string,
  provider: UsageProviderKind,
): Promise<readonly UsageRecord[] | null> {
  const records: UsageRecord[] = [];
  const codexState = initialCodexScanState();

  try {
    const lines = NodeReadline.createInterface({
      input: NodeFS.createReadStream(filePath, { encoding: "utf8" }),
      crlfDelay: Infinity,
    });

    for await (const line of lines) {
      if (provider === "codex") {
        if (
          !mightCarryUsage(line, provider) &&
          !line.includes('"turn_context"') &&
          !line.includes('"session_meta"')
        ) {
          continue;
        }

        const record = parseCodexLine(line, codexState);

        if (record !== null) records.push(record);
        continue;
      }

      if (!mightCarryUsage(line, provider)) continue;
      const record = parseClaudeLine(line);

      if (record !== null) records.push(record);
    }
  } catch {
    return null;
  }

  return records;
}