DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
/**
* Codex adapter. Sessions live at
* `~/.codex/sessions/YYYY/MM/DD/rollout-<ts>-<uuid>.jsonl`, partitioned by date
* rather than project, so we read each file's `session_meta.cwd` and group by
* it ourselves. Older files carry the transcript on the clean `event_msg`
* layer (user_message / agent_message) and are read from there; current ones
* put it in `response_item` messages, whose user side also carries the
* harness's own injected context (see `isHarnessInjectedTurn`). Reasoning and
* sub-agent (`thread_source: "subagent"`) sessions are skipped either way.
*/
import { iterLines, parseJsonl, readHead } from "./streaming";
import {
cleanText,
deriveTitle,
epochMsToISODate,
isoToEpochSeconds,
projectLabel,
} from "./shared";
import type {
NormalizedMessage,
NormalizedSession,
ProjectGroup,
SessionRef,
} from "./types";
const SCAN_HEAD_BYTES = 64 * 1024;
interface CodexLine {
timestamp?: string;
type?: string;
payload?: Record<string, unknown>;
}
function readMeta(lines: CodexLine[]): {
cwd: string;
id: string;
isSubagent: boolean;
} {
const meta = lines.find((l) => l.type === "session_meta")?.payload ?? {};
return {
cwd: typeof meta.cwd === "string" ? meta.cwd : "",
id: typeof meta.id === "string" ? meta.id : "",
isSubagent: meta.thread_source === "subagent",
};
}
function eventMessage(line: CodexLine): NormalizedMessage | null {
if (line.type !== "event_msg") return null;
const p = line.payload ?? {};
let role: "user" | "assistant";
if (p.type === "user_message") role = "user";
else if (p.type === "agent_message") role = "assistant";
else return null;
const content = cleanText(typeof p.message === "string" ? p.message : "");
if (!content) return null;
const created = isoToEpochSeconds(line.timestamp, 0);
return { role, content, created_at: created || undefined };
}
/** Opening tag of a top-level element, capturing its name. */
const OPEN_TAG_RE = /^<([a-z][a-z0-9_-]*)(\s[^>]*)?>/;
/**
* Whether a user turn was written by the harness rather than the person.
*
* Codex delivers its own context to the model as `role: "user"` items —
* `<environment_context>`, `<recommended_plugins>`, `<skill>`, `<task>`,
* `<heartbeat>`, `<turn_aborted>`, `<codex_internal_context>` and more; ten
* distinct tags across 631 local rollouts, about a third of every user turn
* on disk. Imported as-is they become the learner's own words, and the first
* one becomes the session title.
*
* The tag name is not the test — that list only grows, and a new one would
* walk straight through. What every injected turn shares is that it is
* *nothing but* markup: one or more balanced top-level elements with no prose
* of the person's own around them. So: consume top-level elements, and if a
* single character of anything else remains, this is a person's message.
*
* That is the safe direction, and it is doing real work. Codex prefixes an
* attachment turn with `<image name=[Image #1]></image>` and then the actual
* question — 76 such turns here, every one kept, because the question sits
* outside the element. Anything ambiguous (a same-named nested tag, an
* unclosed one, an attribute holding a `>`) falls out of the loop unmatched
* and is likewise kept. Losing a title to a block we failed to recognise is
* a blemish; eating the sentence someone actually typed is not.
*/
function isHarnessInjectedTurn(text: string): boolean {
let rest = text.trim();
let sawElement = false;
while (rest) {
const open = OPEN_TAG_RE.exec(rest);
if (!open) return false;
const closing = `</${open[1]}>`;
const end = rest.indexOf(closing, open[0].length);
if (end === -1) return false;
rest = rest.slice(end + closing.length).trim();
sawElement = true;
}
return sawElement;
}
function responseItemMessage(line: CodexLine): NormalizedMessage | null {
if (line.type !== "response_item") return null;
const p = line.payload ?? {};
if (p.type !== "message" || (p.role !== "user" && p.role !== "assistant")) {
return null;
}
if (!Array.isArray(p.content)) return null;
const parts = p.content.flatMap((item) => {
if (!item || typeof item !== "object") return [];
const block = item as Record<string, unknown>;
if (block.type !== "input_text" && block.type !== "output_text") return [];
return typeof block.text === "string" ? [block.text] : [];
});
const content = cleanText(parts.join("\n\n"));
if (!content) return null;
// Only the user side: an assistant turn is the model's answer either way,
// and it does not carry these blocks.
if (p.role === "user" && isHarnessInjectedTurn(content)) return null;
const created = isoToEpochSeconds(line.timestamp, 0);
return { role: p.role, content, created_at: created || undefined };
}
function preferredMessages(lines: CodexLine[]): NormalizedMessage[] {
const legacy = lines.map(eventMessage).filter((message) => message !== null);
if (legacy.length) return legacy;
return lines.map(responseItemMessage).filter((message) => message !== null);
}
/** A scanned file plus the `YYYY-MM-DD` recovered from its directory trail. */
interface CodexFile {
handle: FileSystemFileHandle;
date: string;
}
/** Turn a `sessions/2026/06/14` directory trail into `2026-06-14`. */
function dateFromTrail(trail: string[]): string {
const nums = trail.filter((seg) => /^\d+$/.test(seg));
if (nums.length < 3) return "";
const [y, m, d] = nums;
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
}
async function walkJsonl(
dir: FileSystemDirectoryHandle,
out: CodexFile[],
trail: string[] = [],
): Promise<void> {
for await (const entry of dir.values()) {
if (entry.kind === "directory") {
await walkJsonl(entry as FileSystemDirectoryHandle, out, [
...trail,
entry.name,
]);
} else if (entry.name.endsWith(".jsonl")) {
out.push({
handle: entry as FileSystemFileHandle,
date: dateFromTrail(trail),
});
}
}
}
export async function scanCodex(
root: FileSystemDirectoryHandle,
): Promise<ProjectGroup[]> {
const sessionsDir = await root.getDirectoryHandle("sessions");
const files: CodexFile[] = [];
await walkJsonl(sessionsDir, files);
const byCwd = new Map<string, SessionRef[]>();
for (const { handle, date } of files) {
const file = await handle.getFile();
const head = parseJsonl(
await readHead(file, SCAN_HEAD_BYTES),
) as CodexLine[];
const meta = readMeta(head);
if (meta.isSubagent) continue;
const cwd = meta.cwd || "(unknown)";
const firstUser = preferredMessages(head).find((m) => m.role === "user");
const ref: SessionRef = {
externalId: meta.id || handle.name.replace(/\.jsonl$/, ""),
provisionalTitle: firstUser ? deriveTitle(firstUser.content) : "",
cwd,
date: date || epochMsToISODate(file.lastModified),
lastModified: file.lastModified,
sizeBytes: file.size,
handle,
};
const arr = byCwd.get(cwd) ?? [];
arr.push(ref);
byCwd.set(cwd, arr);
}
const groups: ProjectGroup[] = [];
for (const [cwd, sessions] of byCwd) {
sessions.sort((a, b) => b.lastModified - a.lastModified);
groups.push({ cwd, label: projectLabel(cwd), sessions });
}
groups.sort(
(a, b) =>
(b.sessions[0]?.lastModified ?? 0) - (a.sessions[0]?.lastModified ?? 0),
);
return groups;
}
export async function parseCodexSession(
ref: SessionRef,
): Promise<NormalizedSession | null> {
const file = await ref.handle.getFile();
const legacyMessages: NormalizedMessage[] = [];
const responseItemMessages: NormalizedMessage[] = [];
let cwd = ref.cwd;
let isSubagent = false;
for await (const line of iterLines(file)) {
let rec: CodexLine;
try {
rec = JSON.parse(line) as CodexLine;
} catch {
continue;
}
if (rec.type === "session_meta") {
const p = rec.payload ?? {};
if (typeof p.cwd === "string") cwd = p.cwd;
if (p.thread_source === "subagent") isSubagent = true;
continue;
}
const legacyMessage = eventMessage(rec);
if (legacyMessage) legacyMessages.push(legacyMessage);
const responseItem = responseItemMessage(rec);
if (responseItem) responseItemMessages.push(responseItem);
}
const messages = legacyMessages.length ? legacyMessages : responseItemMessages;
if (isSubagent || !messages.length) return null;
const fallbackTs = file.lastModified / 1000;
const firstTs = messages.find((m) => m.created_at)?.created_at ?? fallbackTs;
const lastTs =
[...messages].reverse().find((m) => m.created_at)?.created_at ?? fallbackTs;
return {
external_id: ref.externalId,
title: ref.provisionalTitle || deriveTitle(messages[0].content),
source_cwd: cwd,
created_at: firstTs,
updated_at: lastTs,
messages,
};
}