Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

19.1 KB
Raw Blame
#!/usr/bin/env node

/**
 * SessionStart hook for Codex.
 *
 * Triggers (matcher = "clear|startup|resume" in hooks.json):
 *   - source=startup → fresh codex CLI / `/new` / zouk daemon spawn-without-sessionId
 *   - source=clear   → `/clear` (orphans the current process's previous session)
 *   - source=resume  → `/resume` or short reconnect (no commit/sweep;
 *     may inject latest archive summary if the live OV session was already committed)
 *
 * Every source injects the shared OpenViking profile block unless
 * OPENVIKING_NO_AUTO_INJECT=1. The block contains profile.md plus
 * abstract-annotated indexes of preferences/ and entities/, capped by
 * OPENVIKING_PROFILE_TOKEN_BUDGET with the shared CJK-aware estimator.
 *
 * Behavior (see DESIGN.md — "Fallback sweep"):
 *   Committing a finished thread is SessionEnd's job. On `startup` or `clear`
 *   this hook only sweeps what SessionEnd could not reach: a state file with a
 *   live OV session is committed when it carries an `.ended` marker (the
 *   SessionEnd worker was killed or the server was down) or when it has been
 *   idle for more than IDLE_TTL_MS (default 30 min) — signals, crashes, Codex
 *   older than 0.145, and app-server threads whose SessionEnd is deferred.
 *
 *   Each candidate is committed under its session lock with no waiting: a lock
 *   we cannot take means a SessionEnd or Stop worker already owns the session.
 *   Under the lock the sweep first appends whatever the state's recorded
 *   transcript still holds past the cursor, so a session whose workers never
 *   ran is not archived without its tail turns; an incomplete append keeps the
 *   session live for the next sweep instead of committing.
 *
 *   The same pass retires cursor-only states (no live OV session): a cursor
 *   that was never used is dropped after IDLE_TTL_MS, and a real cursor is
 *   kept for resume until COMMITTED_TTL_MS (default 30 days). Without this the
 *   state directory grows one file per codex session forever and listStates()
 *   reads all of them on every SessionStart.
 *
 * Commit failure handling:
 *   On any /commit failure (OV unreachable, non-2xx, timeout) we keep the state
 *   file with ovSessionId still set so the next sweep retries. A transient OV
 *   outage shouldn't lose memory.
 *
 * Output may contain hookSpecificOutput.additionalContext for profile/archive
 * injection and systemMessage for commit status at the same time.
 */

import { join } from "node:path";

import { loadConfig } from "./config.mjs";
import { createLogger } from "./debug-log.mjs";
import { catchUpTurns, commitOvSession, makeFetchJSON } from "./ov-session.mjs";
import { detectRecallCompressorProfile } from "./recall-compressor-profile.mjs";
import {
  clearEnded,
  clearState,
  deriveOvSessionId,
  getStateDir,
  listStates,
  loadState,
  readEndedAt,
  saveState,
  withSessionLock,
} from "./session-state.mjs";
import { runHookStage } from "./shared/agent-hook-runtime.mjs";
import { replayPending } from "./shared/pending-queue.mjs";
import { buildProfileBlock, isRepeatInjection, truncateToBytes } from "./shared/profile-inject.mjs";
import { resolveEffectivePeerId } from "./shared/workspace-peer.mjs";

let cfg = loadConfig();
const { log, logError } = createLogger("session-start");
let activePeerId = cfg.peerId || "";

const IDLE_TTL_MS = (() => {
  const v = Number(process.env.OPENVIKING_CODEX_IDLE_TTL_MS);
  return Number.isFinite(v) && v > 0 ? Math.floor(v) : 1_800_000;
})();

const HOOK_STARTED_AT = Date.now();

// The sweep catches up unsent turns before committing, so it needs the same
// HTTP helper the capture hooks use.
const { fetchJSONRes, fetchJSON } = makeFetchJSON(cfg, { getActorPeerId: () => activePeerId });

const COMMITTED_TTL_MS = (() => {
  const v = Number(process.env.OPENVIKING_CODEX_COMMITTED_TTL_MS);
  return Number.isFinite(v) && v > 0 ? Math.floor(v) : 2_592_000_000;
})();

function output(obj) {
  process.stdout.write(JSON.stringify(obj) + "\n");
}

function noop(message) {
  output(message ? { systemMessage: message } : {});
}

function emitSessionStartOutput({ contexts = [], systemMessage = "" } = {}) {
  const additionalContext = contexts.filter(Boolean).join("\n\n");
  const response = {};
  if (additionalContext) {
    response.hookSpecificOutput = {
      hookEventName: "SessionStart",
      additionalContext,
    };
  }
  if (systemMessage) response.systemMessage = systemMessage;
  output(response);
}

/**
 * Drain writes that an earlier hook queued while the server was unreachable.
 * SessionStart is the only codex hook that runs after a known-healthy check,
 * so it is where the queue gets its chance; a bypassed directory still replays,
 * because the entries were recorded by sessions that were not bypassed.
 */
async function replayPendingWrites() {
  try {
    const result = await replayPending(fetchJSONRes, log);
    if (result.replayed > 0 || result.failed > 0 || result.deferred > 0) {
      log("pending-replay", result);
    }
  } catch (err) {
    logError("pending-replay", err);
  }
}

function truncateText(text, maxChars) {
  const value = String(text || "").trim();
  if (value.length <= maxChars) return value;
  return `${value.slice(0, Math.max(0, maxChars - 20)).trimEnd()}\n[truncated]`;
}

function formatResumeArchiveContext(ovSessionId, context) {
  const overview = String(context?.latest_archive_overview || "").trim();
  if (!overview) return "";
  const archiveUri = `viking://~/sessions/${ovSessionId}/history/`;
  const head = [
    "OpenViking session archive digest:",
    `Latest committed archive for resumed Codex session ${ovSessionId}:`,
  ];
  const tail = [
    "",
    `More detail: use the OpenViking MCP read/search tools with ${archiveUri} if you need exact prior commands, files, tool outputs, or messages.`,
  ];
  // Under a byte cap the archive takes at most half of the SessionStart context.
  const maxBytes = cfg.sessionStartMaxBytes > 0
    ? Math.max(1, Math.floor(cfg.sessionStartMaxBytes / 2) - ENVELOPE_BYTES
      - Buffer.byteLength([...head, ...tail].join("\n"), "utf8"))
    : 0;
  const body = truncateToBytes(truncateText(overview, cfg.resumeArchiveMaxChars), maxBytes);
  return [...head, body, ...tail].join("\n");
}

function wrapResumeContext(additionalContext) {
  const body = String(additionalContext || "")
    .replace(/<\/?openviking-context\b[^>]*>/gi, "openviking context marker")
    .trim();
  if (!body) return "";
  return [
    '<openviking-context source="session-resume" format="archive-digest">',
    body,
    "</openviking-context>",
  ].join("\n");
}

function wrapProfileContext(profileBlock) {
  if (!profileBlock) return "";
  return [
    '<openviking-context source="session-start">',
    profileBlock,
    "</openviking-context>",
  ].join("\n");
}

// Leaves room for the envelope lines and the separator between contexts.
const ENVELOPE_BYTES = 100;

async function buildSessionProfileContext({ sessionId = "", source = "", maxBytes = cfg.sessionStartMaxBytes } = {}) {
  if (cfg.noAutoInject) {
    log("skip", { stage: "profile_inject", reason: "disabled" });
    return "";
  }
  try {
    const profile = await buildProfileBlock(
      fetchJSONRes,
      cfg.profileTokenBudget,
      activePeerId,
      { ...cfg, sessionStartMaxBytes: maxBytes > 0 ? Math.max(1, maxBytes - ENVELOPE_BYTES) : 0 },
    );
    if (!profile?.block) {
      log("skip", { stage: "profile_inject", reason: "no profile content" });
      return "";
    }
    // A resumed thread already holds the earlier block; skip it when unchanged.
    const statePath = join(getStateDir(), "profile-injections.json");
    if (sessionId && isRepeatInjection(statePath, sessionId, profile.block) && source === "resume") {
      log("skip", { stage: "profile_inject", reason: "unchanged since last injection" });
      return "";
    }
    log("profile_inject", {
      chars: profile.chars,
      tokens: profile.tokens,
      profileChars: profile.profileChars,
      prefCount: profile.prefCount,
      entCount: profile.entCount,
      droppedPref: profile.droppedPref,
      droppedEnt: profile.droppedEnt,
      skillCount: profile.skillCount,
      droppedSkill: profile.droppedSkill,
      skillTokens: profile.skillTokens,
    });
    return wrapProfileContext(profile.block);
  } catch (error) {
    logError("profile_inject", error);
    return "";
  }
}

async function buildResumeArchiveContext(newSessionId) {
  if (!cfg.resumeArchiveInject) {
    log("skip", { stage: "resume_archive", reason: "disabled" });
    return "";
  }

  const state = await loadState(newSessionId);
  if (state.ovSessionId) {
    log("skip", {
      stage: "resume_archive",
      reason: "live OV session still open",
      ovSessionId: state.ovSessionId,
    });
    return "";
  }

  const ovSessionId = deriveOvSessionId(newSessionId);
  const context = await fetchJSON(
    `/api/v1/sessions/${encodeURIComponent(ovSessionId)}/context?token_budget=${cfg.resumeArchiveTokenBudget}`,
  );
  const additionalContext = formatResumeArchiveContext(ovSessionId, context);
  if (!additionalContext) {
    log("skip", { stage: "resume_archive", reason: "no archive overview", ovSessionId });
    return "";
  }

  log("resume_archive_inject", {
    ovSessionId,
    chars: additionalContext.length,
    tokenBudget: cfg.resumeArchiveTokenBudget,
  });
  return wrapResumeContext(additionalContext);
}

/**
 * Commit a live OV session and release it, keeping the transcript cursor so a
 * later resume appends instead of replaying (DESIGN.md "Commit-then-resume").
 * On commit failure, keep the live session id so the next sweep retries.
 *
 * Returns { committed: bool, ovSessionId: string|null, traceId: string }.
 */
async function commitAndRelease(state, reason, endToken) {
  const ovSessionId = state.ovSessionId;
  const commit = await commitOvSession(fetchJSONRes, ovSessionId);
  if (!commit?.ok) {
    log("commit", {
      reason,
      codexSessionId: state.codexSessionId,
      ovSessionId,
      ok: false,
      status: commit?.status,
      trace_id: commit?.traceId,
      error: commit?.error?.message || commit?.error?.code,
    });
    return { committed: false, ovSessionId: null, traceId: commit?.traceId || "" };
  }
  const traceId = commit.traceId || commit.result?.trace_id || "";
  log("commit", {
    reason,
    codexSessionId: state.codexSessionId,
    ovSessionId,
    archived: commit.result?.archived ?? false,
    taskId: commit.result?.task_id,
    status: commit.result?.status,
    trace_id: traceId || undefined,
  });
  state.ovSessionId = null;
  await saveState(state, { touch: false });
  // Only retire the marker this pass verified; a newer one belongs to an exit
  // that happened while we were committing.
  await clearEnded(
    state.codexSessionId,
    typeof endToken === "number" ? { before: endToken + 1 } : {},
  );
  return { committed: true, ovSessionId, traceId };
}

/**
 * Retire a state file with no live OV session. A cursor that never captured
 * anything carries no resume value, so it goes on the idle schedule; a real
 * cursor is kept until COMMITTED_TTL_MS in case the codex session is resumed.
 */
async function maybeRetireCursorState(state, ageMs) {
  const hasCursor = Number(state.capturedTurnCount) > 0;
  const ttl = hasCursor ? COMMITTED_TTL_MS : IDLE_TTL_MS;
  if (ageMs <= ttl) return false;
  log("state_retire", {
    codexSessionId: state.codexSessionId,
    capturedTurnCount: state.capturedTurnCount,
    ageMs,
    ttlMs: ttl,
  });
  await clearState(state.codexSessionId);
  return true;
}

function describeCommittedSessions(commits) {
  const traceIds = commits.map((item) => item.traceId).filter(Boolean);
  if (commits.length === 1) {
    return `OpenViking session ${commits[0].ovSessionId} is committed` +
      (traceIds.length ? ` (trace_id=${traceIds[0]})` : "");
  }
  return `OpenViking sessions ${commits.map((item) => item.ovSessionId).join(", ")} are committed` +
    (traceIds.length ? ` (trace_ids=${traceIds.join(",")})` : "");
}

// A bypassed directory suppresses this session's own memory work — no peer
// registration, no injection. The sweep and the pending replay still run: they
// finish sessions recorded elsewhere, and this hook is the only place codex
// runs either, so skipping them would strand that data for as long as the user
// keeps working in a bypassed repository.
runHookStage({
  loadConfig,
  gates: { bypass: () => false },
  envelope: (response) => emitSessionStartOutput(response || {}),
  onSkip: (reason) => log("skip", { stage: "init", reason }),
}, async (stage) => {
  const { input, cwd, bypassed } = stage;
  cfg = stage.cfg;
  const source = input.source || "unknown";
  const newSessionId = input.session_id || "unknown";
  const effectivePeer = resolveEffectivePeerId({ cfg, cwd });
  activePeerId = effectivePeer.peerId;
  if (!bypassed && newSessionId !== "unknown") {
    const state = await loadState(newSessionId);
    await saveState({
      ...state,
      workspacePeerId: effectivePeer.source === "workspace" ? effectivePeer.peerId : "",
    });
  }
  log("start", {
    source,
    newSessionId,
    idleTtlMs: IDLE_TTL_MS,
    peerSource: effectivePeer.source,
    bypassed,
  });

  try {
    await detectRecallCompressorProfile(cfg, { log, logError });
  } catch (err) {
    logError("compress_profile_detect_uncaught", err);
  }

  if (source === "resume") {
    // Earliest liveness signal after `codex resume`: the thread is running
    // again, so a marker left by its previous exit must not trigger the sweep.
    await clearEnded(newSessionId, { before: HOOK_STARTED_AT });
    const health = await fetchJSON("/health");
    if (!health) {
      logError("health_check", "server unreachable; skipping profile + archive injection");
      return;
    }
    await replayPendingWrites();
    if (bypassed) {
      log("skip", { stage: "inject", reason: "bypass_session_pattern" });
      return;
    }
    const archiveContext = await buildResumeArchiveContext(newSessionId);
    const maxBytes = cfg.sessionStartMaxBytes > 0
      ? Math.max(1, cfg.sessionStartMaxBytes - Buffer.byteLength(archiveContext, "utf8"))
      : 0;
    const profileContext = await buildSessionProfileContext({ sessionId: newSessionId, source, maxBytes });
    return { contexts: [profileContext, archiveContext] };
  }

  // Other non-startup sources are hard no-ops. We don't sweep there, because
  // reconnect-like sources may fire often and sweep should stay tied to a new
  // session boundary.
  if (source !== "startup" && source !== "clear") {
    log("skip", { stage: "source_check", reason: `source=${source} (only startup|clear act)` });
    return;
  }

  const health = await fetchJSON("/health");
  if (!health) {
    logError("health_check", "server unreachable; skipping profile injection + commit + sweep");
    return;
  }

  await replayPendingWrites();

  const profileContext = bypassed
    ? null
    : await buildSessionProfileContext({ sessionId: newSessionId, source });
  const now = Date.now();
  const commits = [];
  let retired = 0;
  let lockSkipped = 0;

  for (const s of await listStates()) {
    if (!s?.codexSessionId) continue;
    if (s.codexSessionId === newSessionId) continue;
    if (typeof s.lastUpdatedAt !== "number") continue;
    const ageMs = now - s.lastUpdatedAt;

    // A marker must always enter the lock, even with no live id: PreCompact
    // releases the id while leaving a cursor behind, so the tail turns of a
    // session whose SessionEnd worker died are only reachable through a
    // catch-up under the lock.
    if (!s.ovSessionId && !s.endedAt) {
      if (await maybeRetireCursorState(s, ageMs)) retired += 1;
      continue;
    }
    if (!s.endedAt && ageMs <= IDLE_TTL_MS) continue;

    const reason = s.endedAt ? "ended_retry" : "idle_ttl";
    log("sweep", { codexSessionId: s.codexSessionId, ovSessionId: s.ovSessionId, ageMs, reason });
    // Try-lock only: a held lock means a SessionEnd or Stop worker is already
    // committing this session (user quit and relaunched within seconds).
    const outcome = await withSessionLock(s.codexSessionId, async ({ heartbeat }) => {
      const fresh = await loadState(s.codexSessionId);

      // Re-read the marker under the lock: it may have been cleared by a
      // resume or replaced by a newer exit since listStates() sampled it.
      const endToken = await readEndedAt(s.codexSessionId);
      if (reason === "ended_retry" && (endToken === undefined || endToken > s.endedAt)) {
        if (ageMs <= IDLE_TTL_MS) {
          log("sweep_skip", {
            codexSessionId: s.codexSessionId,
            reason: endToken === undefined ? "marker cleared under the lock" : "newer end marker",
          });
          return null;
        }
      }

      // Turns the session's own workers never sent would be lost by an
      // archive-now commit, so catch them up first and keep the session live
      // for the next sweep if any of them failed to land.
      const { newTurns, added, skipped, unreadable } = await catchUpTurns({
        state: fresh,
        transcriptPath: fresh.transcriptPath,
        fetchJSONRes,
        activePeerId: fresh.workspacePeerId || activePeerId,
        cfg,
        log,
        logError,
        heartbeat,
      });
      if (added > 0) log("appended_catchup", { ovSessionId: fresh.ovSessionId, added });

      // An unreadable transcript is not an empty one: the tail turns may still
      // be there. Keep the live id and the marker for the next sweep.
      if (unreadable) {
        logError("transcript_unreadable", {
          codexSessionId: s.codexSessionId,
          ovSessionId: fresh.ovSessionId,
          transcriptPath: fresh.transcriptPath,
        });
        await saveState(fresh, { touch: false });
        return null;
      }

      if (newTurns.length > 0 && !skipped && added < newTurns.length) {
        logError("append_incomplete", {
          codexSessionId: s.codexSessionId,
          ovSessionId: fresh.ovSessionId,
          attempted: newTurns.length,
          added,
        });
        await saveState(fresh, { touch: false });
        return null;
      }

      // The catch-up derives a live id whenever it sends something; still
      // having none means there is genuinely nothing to commit, so the marker
      // can go.
      if (!fresh.ovSessionId) {
        if (added > 0) await saveState(fresh, { touch: false });
        await clearEnded(
          s.codexSessionId,
          typeof endToken === "number" ? { before: endToken + 1 } : {},
        );
        log("skip", {
          stage: "commit",
          codexSessionId: s.codexSessionId,
          reason: "no live OV session for this codex session",
        });
        return null;
      }

      return commitAndRelease(fresh, reason, endToken);
    }, { waitMs: 0 });

    if (outcome.skipped) {
      lockSkipped += 1;
      log("sweep_skip", { codexSessionId: s.codexSessionId, reason: "locked by another writer" });
      continue;
    }
    if (outcome.value?.committed) commits.push(outcome.value);
  }

  const ovSessionIds = commits.map((item) => item.ovSessionId);

  log("done", {
    source,
    committed: commits.length,
    retired,
    lockSkipped,
    ovSessionIds,
  });

  return {
    contexts: [profileContext],
    systemMessage: commits.length > 0 ? describeCommittedSessions(commits) : "",
  };
}).catch((err) => { logError("uncaught", err); noop(); });