Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

8.7 KB
Raw Blame
/**
 * Codex adapter. Sessions live at
 * `~/.codex/sessions/YYYY/MM/DD/rollout-<ts>-<uuid>.jsonl`, partitioned by date
 * rather than project, so we read each file's `session_meta.cwd` and group by
 * it ourselves. Older files carry the transcript on the clean `event_msg`
 * layer (user_message / agent_message) and are read from there; current ones
 * put it in `response_item` messages, whose user side also carries the
 * harness's own injected context (see `isHarnessInjectedTurn`). Reasoning and
 * sub-agent (`thread_source: "subagent"`) sessions are skipped either way.
 */

import { iterLines, parseJsonl, readHead } from "./streaming";
import {
  cleanText,
  deriveTitle,
  epochMsToISODate,
  isoToEpochSeconds,
  projectLabel,
} from "./shared";
import type {
  NormalizedMessage,
  NormalizedSession,
  ProjectGroup,
  SessionRef,
} from "./types";

const SCAN_HEAD_BYTES = 64 * 1024;

interface CodexLine {
  timestamp?: string;
  type?: string;
  payload?: Record<string, unknown>;
}

function readMeta(lines: CodexLine[]): {
  cwd: string;
  id: string;
  isSubagent: boolean;
} {
  const meta = lines.find((l) => l.type === "session_meta")?.payload ?? {};
  return {
    cwd: typeof meta.cwd === "string" ? meta.cwd : "",
    id: typeof meta.id === "string" ? meta.id : "",
    isSubagent: meta.thread_source === "subagent",
  };
}

function eventMessage(line: CodexLine): NormalizedMessage | null {
  if (line.type !== "event_msg") return null;
  const p = line.payload ?? {};
  let role: "user" | "assistant";
  if (p.type === "user_message") role = "user";
  else if (p.type === "agent_message") role = "assistant";
  else return null;
  const content = cleanText(typeof p.message === "string" ? p.message : "");
  if (!content) return null;
  const created = isoToEpochSeconds(line.timestamp, 0);
  return { role, content, created_at: created || undefined };
}

/** Opening tag of a top-level element, capturing its name. */
const OPEN_TAG_RE = /^<([a-z][a-z0-9_-]*)(\s[^>]*)?>/;

/**
 * Whether a user turn was written by the harness rather than the person.
 *
 * Codex delivers its own context to the model as `role: "user"` items —
 * `<environment_context>`, `<recommended_plugins>`, `<skill>`, `<task>`,
 * `<heartbeat>`, `<turn_aborted>`, `<codex_internal_context>` and more; ten
 * distinct tags across 631 local rollouts, about a third of every user turn
 * on disk. Imported as-is they become the learner's own words, and the first
 * one becomes the session title.
 *
 * The tag name is not the test — that list only grows, and a new one would
 * walk straight through. What every injected turn shares is that it is
 * *nothing but* markup: one or more balanced top-level elements with no prose
 * of the person's own around them. So: consume top-level elements, and if a
 * single character of anything else remains, this is a person's message.
 *
 * That is the safe direction, and it is doing real work. Codex prefixes an
 * attachment turn with `<image name=[Image #1]></image>` and then the actual
 * question — 76 such turns here, every one kept, because the question sits
 * outside the element. Anything ambiguous (a same-named nested tag, an
 * unclosed one, an attribute holding a `>`) falls out of the loop unmatched
 * and is likewise kept. Losing a title to a block we failed to recognise is
 * a blemish; eating the sentence someone actually typed is not.
 */
function isHarnessInjectedTurn(text: string): boolean {
  let rest = text.trim();
  let sawElement = false;
  while (rest) {
    const open = OPEN_TAG_RE.exec(rest);
    if (!open) return false;
    const closing = `</${open[1]}>`;
    const end = rest.indexOf(closing, open[0].length);
    if (end === -1) return false;
    rest = rest.slice(end + closing.length).trim();
    sawElement = true;
  }
  return sawElement;
}

function responseItemMessage(line: CodexLine): NormalizedMessage | null {
  if (line.type !== "response_item") return null;
  const p = line.payload ?? {};
  if (p.type !== "message" || (p.role !== "user" && p.role !== "assistant")) {
    return null;
  }
  if (!Array.isArray(p.content)) return null;

  const parts = p.content.flatMap((item) => {
    if (!item || typeof item !== "object") return [];
    const block = item as Record<string, unknown>;
    if (block.type !== "input_text" && block.type !== "output_text") return [];
    return typeof block.text === "string" ? [block.text] : [];
  });
  const content = cleanText(parts.join("\n\n"));
  if (!content) return null;
  // Only the user side: an assistant turn is the model's answer either way,
  // and it does not carry these blocks.
  if (p.role === "user" && isHarnessInjectedTurn(content)) return null;
  const created = isoToEpochSeconds(line.timestamp, 0);
  return { role: p.role, content, created_at: created || undefined };
}

function preferredMessages(lines: CodexLine[]): NormalizedMessage[] {
  const legacy = lines.map(eventMessage).filter((message) => message !== null);
  if (legacy.length) return legacy;
  return lines.map(responseItemMessage).filter((message) => message !== null);
}

/** A scanned file plus the `YYYY-MM-DD` recovered from its directory trail. */
interface CodexFile {
  handle: FileSystemFileHandle;
  date: string;
}

/** Turn a `sessions/2026/06/14` directory trail into `2026-06-14`. */
function dateFromTrail(trail: string[]): string {
  const nums = trail.filter((seg) => /^\d+$/.test(seg));
  if (nums.length < 3) return "";
  const [y, m, d] = nums;
  return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
}

async function walkJsonl(
  dir: FileSystemDirectoryHandle,
  out: CodexFile[],
  trail: string[] = [],
): Promise<void> {
  for await (const entry of dir.values()) {
    if (entry.kind === "directory") {
      await walkJsonl(entry as FileSystemDirectoryHandle, out, [
        ...trail,
        entry.name,
      ]);
    } else if (entry.name.endsWith(".jsonl")) {
      out.push({
        handle: entry as FileSystemFileHandle,
        date: dateFromTrail(trail),
      });
    }
  }
}

export async function scanCodex(
  root: FileSystemDirectoryHandle,
): Promise<ProjectGroup[]> {
  const sessionsDir = await root.getDirectoryHandle("sessions");
  const files: CodexFile[] = [];
  await walkJsonl(sessionsDir, files);

  const byCwd = new Map<string, SessionRef[]>();
  for (const { handle, date } of files) {
    const file = await handle.getFile();
    const head = parseJsonl(
      await readHead(file, SCAN_HEAD_BYTES),
    ) as CodexLine[];
    const meta = readMeta(head);
    if (meta.isSubagent) continue;
    const cwd = meta.cwd || "(unknown)";
    const firstUser = preferredMessages(head).find((m) => m.role === "user");
    const ref: SessionRef = {
      externalId: meta.id || handle.name.replace(/\.jsonl$/, ""),
      provisionalTitle: firstUser ? deriveTitle(firstUser.content) : "",
      cwd,
      date: date || epochMsToISODate(file.lastModified),
      lastModified: file.lastModified,
      sizeBytes: file.size,
      handle,
    };
    const arr = byCwd.get(cwd) ?? [];
    arr.push(ref);
    byCwd.set(cwd, arr);
  }

  const groups: ProjectGroup[] = [];
  for (const [cwd, sessions] of byCwd) {
    sessions.sort((a, b) => b.lastModified - a.lastModified);
    groups.push({ cwd, label: projectLabel(cwd), sessions });
  }
  groups.sort(
    (a, b) =>
      (b.sessions[0]?.lastModified ?? 0) - (a.sessions[0]?.lastModified ?? 0),
  );
  return groups;
}

export async function parseCodexSession(
  ref: SessionRef,
): Promise<NormalizedSession | null> {
  const file = await ref.handle.getFile();
  const legacyMessages: NormalizedMessage[] = [];
  const responseItemMessages: NormalizedMessage[] = [];
  let cwd = ref.cwd;
  let isSubagent = false;

  for await (const line of iterLines(file)) {
    let rec: CodexLine;
    try {
      rec = JSON.parse(line) as CodexLine;
    } catch {
      continue;
    }
    if (rec.type === "session_meta") {
      const p = rec.payload ?? {};
      if (typeof p.cwd === "string") cwd = p.cwd;
      if (p.thread_source === "subagent") isSubagent = true;
      continue;
    }
    const legacyMessage = eventMessage(rec);
    if (legacyMessage) legacyMessages.push(legacyMessage);
    const responseItem = responseItemMessage(rec);
    if (responseItem) responseItemMessages.push(responseItem);
  }

  const messages = legacyMessages.length ? legacyMessages : responseItemMessages;
  if (isSubagent || !messages.length) return null;
  const fallbackTs = file.lastModified / 1000;
  const firstTs = messages.find((m) => m.created_at)?.created_at ?? fallbackTs;
  const lastTs =
    [...messages].reverse().find((m) => m.created_at)?.created_at ?? fallbackTs;
  return {
    external_id: ref.externalId,
    title: ref.provisionalTitle || deriveTitle(messages[0].content),
    source_cwd: cwd,
    created_at: firstTs,
    updated_at: lastTs,
    messages,
  };
}