Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

17.8 KB
Raw Blame
#!/usr/bin/env node

/**
 * LoCoMo benchmark companion to the plugin's auto-capture.mjs.
 *
 * Difference from upstream:
 *   pushTurnsToOv() extracts the LoCoMo session_time stamp from the user
 *   prompt (e.g. "[group chat conversation: 4:02 pm on 12 February, 2023]"
 *   or "=== session_3 (...) ===") and passes a per-message `created_at`
 *   to OV addMessage, mirroring the SDK ingest path (import_to_ov.py).
 *
 * Why: the OV server derives event archive dates from message.created_at.
 * Without this patch, e2e auto-captured LoCoMo conversations all stamp
 * under the ingest day, collapsing the temporal dimension that semantic
 * search relies on.
 *
 * Behaviour outside LoCoMo prompts is unchanged: regex miss → omit
 * created_at → server falls back to now() (upstream behaviour).
 *
 * The upstream plugin's lib is loaded dynamically from
 * $OPENVIKING_PLUGIN_DIR so this script lives outside the plugin tree and
 * the plugin source is untouched.
 */

import { readFile, writeFile, mkdir } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { pathToFileURL } from "node:url";

const pluginDir = process.env.OPENVIKING_PLUGIN_DIR;
if (!pluginDir) {
  process.stderr.write("auto-capture: OPENVIKING_PLUGIN_DIR not set\n");
  process.stdout.write(JSON.stringify({ decision: "approve" }) + "\n");
  process.exit(0);
}
const PLUGIN_SCRIPTS = pathToFileURL(`${pluginDir}/scripts`).href;

const { isPluginEnabled, loadConfig } = await import(`${PLUGIN_SCRIPTS}/config.mjs`);
const { createLogger } = await import(`${PLUGIN_SCRIPTS}/debug-log.mjs`);
const {
  addMessage,
  commitSession,
  deriveOvSessionId,
  getSession,
  isBypassed,
  makeFetchJSON,
} = await import(`${PLUGIN_SCRIPTS}/lib/ov-session.mjs`);
const { maybeDetach, readHookStdin } = await import(`${PLUGIN_SCRIPTS}/lib/async-writer.mjs`);
const { readJsonState, writeJsonState } = await import(`${PLUGIN_SCRIPTS}/lib/state.mjs`);

if (!isPluginEnabled()) {
  process.stdout.write(JSON.stringify({ decision: "approve" }) + "\n");
  process.exit(0);
}

const cfg = loadConfig();
const { log, logError } = createLogger("auto-capture-locomo-e2e");
const fetchJSON = makeFetchJSON(cfg, "captureTimeoutMs");

const STATE_DIR = join(tmpdir(), "openviking-cc-capture-state");

function output(obj) {
  process.stdout.write(JSON.stringify(obj) + "\n");
}

function approve(msg) {
  const out = { decision: "approve" };
  if (msg) out.systemMessage = msg;
  output(out);
}

function stateFilePath(sessionId) {
  const safe = sessionId.replace(/[^a-zA-Z0-9_-]/g, "_");
  return join(STATE_DIR, `${safe}.json`);
}

async function loadState(sessionId) {
  try {
    const data = await readFile(stateFilePath(sessionId), "utf-8");
    return JSON.parse(data);
  } catch {
    return { capturedTurnCount: 0 };
  }
}

async function saveState(sessionId, state) {
  try {
    await mkdir(STATE_DIR, { recursive: true });
    await writeFile(stateFilePath(sessionId), JSON.stringify(state));
  } catch { /* best effort */ }
}

// ---------------------------------------------------------------------------
// Text processing (verbatim from upstream)
// ---------------------------------------------------------------------------

const MEMORY_TRIGGERS = [
  /remember|preference|prefer|important|decision|decided|always|never/i,
  /记住|偏好|喜欢|喜爱|崇拜|讨厌|害怕|重要|决定|总是|永远|优先|习惯|爱好|擅长|最爱|不喜欢/i,
  /[\w.-]+@[\w.-]+\.\w+/,
  /\+\d{10,}/,
  /(?:我|my)\s*(?:是|叫|名字|name|住在|live|来自|from|生日|birthday|电话|phone|邮箱|email)/i,
  /(?:我|i)\s*(?:喜欢|崇拜|讨厌|害怕|擅长|不会|爱|恨|想要|需要|希望|觉得|认为|相信)/i,
  /(?:favorite|favourite|love|hate|enjoy|dislike|admire|idol|fan of)/i,
];

const RELEVANT_MEMORIES_BLOCK_RE = /<relevant-memories>[\s\S]*?<\/relevant-memories>/gi;
const OPENVIKING_CTX_BLOCK_RE = /<openviking-context>[\s\S]*?<\/openviking-context>/gi;
const SYSTEM_REMINDER_BLOCK_RE = /<system-reminder>[\s\S]*?<\/system-reminder>/gi;
const SUBAGENT_CONTEXT_LINE_RE = /^\[Subagent Context\][^\n]*$/gmi;
const COMMAND_TEXT_RE = /^\/[a-z0-9_-]{1,64}\b/i;
const NON_CONTENT_TEXT_RE = /^[\p{P}\p{S}\s]+$/u;
const CJK_CHAR_RE = /[぀-ヿ㐀-鿿豈-﫿가-힯]/;
const QUESTION_ONLY_RE = /^(who|what|when|where|why|how|is|are|does|did|can|could|would|should|may|might|will|谁|什么|何|哪|为什么|怎么|如何|是|会|能|能否)\b.{0,200}[??]$/i;

function stripInjectedBlocks(text) {
  return text
    .replace(RELEVANT_MEMORIES_BLOCK_RE, "")
    .replace(OPENVIKING_CTX_BLOCK_RE, "")
    .replace(SYSTEM_REMINDER_BLOCK_RE, "")
    .replace(SUBAGENT_CONTEXT_LINE_RE, "")
    .replace(/\x00/g, "");
}

// CC's `claude -p --resume <id>` injects synthetic bridging turns:
//   user: "Continue from where you left off."
//   assistant: "No response requested."
// These are not real conversation content and contaminate per-message
// ingest if passed to OV — filter them out here.
const CC_RESUME_BRIDGE_RE = /^(continue from where you left off\.?|no response requested\.?)$/i;
function isCcResumeBridge(text) {
  if (!text) return false;
  const trimmed = text.trim();
  if (trimmed.length > 60) return false;
  return CC_RESUME_BRIDGE_RE.test(trimmed);
}

function sanitize(text) {
  return stripInjectedBlocks(text)
    .replace(/\s+/g, " ")
    .trim();
}

// ---------------------------------------------------------------------------
// LoCoMo session_time extraction (the only behavioural delta from upstream)
// ---------------------------------------------------------------------------

const MONTHS = {
  january: 0, february: 1, march: 2, april: 3, may: 4, june: 5,
  july: 6, august: 7, september: 8, october: 9, november: 10, december: 11,
};

// Matches LoCoMo session_X_date_time strings like "7:55 pm on 9 June, 2023"
// whether wrapped in `(...)` (v12 prompt) or `[...]` (v14 group-chat header)
// or appearing bare. Boundaries are loose because LoCoMo prompts vary.
const LOCOMO_TS_RE =
  /(\d{1,2}):(\d{2})\s*([ap]m)\s+on\s+(\d{1,2})\s+([A-Za-z]+),\s*(\d{4})/i;

function parseLocomoStamp(text) {
  const m = text.match(LOCOMO_TS_RE);
  if (!m) return null;
  let h = parseInt(m[1], 10);
  const min = parseInt(m[2], 10);
  const ampm = m[3].toLowerCase();
  if (ampm === "pm" && h < 12) h += 12;
  if (ampm === "am" && h === 12) h = 0;
  const day = parseInt(m[4], 10);
  const mo = MONTHS[m[5].toLowerCase()];
  const year = parseInt(m[6], 10);
  if (mo === undefined) return null;
  // Use UTC to avoid host TZ skewing the day boundary.
  return new Date(Date.UTC(year, mo, day, h, min, 0));
}

function extractBaseDate(turns) {
  for (const t of turns) {
    if (t.role !== "user") continue;
    const d = parseLocomoStamp(t.text || "");
    if (d) return d;
  }
  return null;
}

function shouldCapture(text) {
  const normalized = sanitize(text);
  if (!normalized) return { capture: false, reason: "empty", text: "" };

  const compact = normalized.replace(/\s+/g, "");
  const minLen = CJK_CHAR_RE.test(compact) ? 4 : 10;
  if (compact.length < minLen || normalized.length > cfg.captureMaxLength) {
    return { capture: false, reason: "length_out_of_range", text: normalized };
  }

  if (COMMAND_TEXT_RE.test(normalized)) {
    return { capture: false, reason: "command", text: normalized };
  }

  if (NON_CONTENT_TEXT_RE.test(normalized)) {
    return { capture: false, reason: "non_content", text: normalized };
  }

  if (QUESTION_ONLY_RE.test(normalized)) {
    return { capture: false, reason: "question_only", text: normalized };
  }

  if (cfg.captureMode === "keyword") {
    for (const trigger of MEMORY_TRIGGERS) {
      if (trigger.test(normalized)) {
        return { capture: true, reason: `trigger:${trigger}`, text: normalized };
      }
    }
    return { capture: false, reason: "no_trigger", text: normalized };
  }

  return { capture: true, reason: "semantic", text: normalized };
}

// ---------------------------------------------------------------------------
// Transcript parsing (verbatim from upstream)
// ---------------------------------------------------------------------------

function parseTranscript(content) {
  try {
    const data = JSON.parse(content);
    if (Array.isArray(data)) return data;
  } catch { /* not a JSON array */ }

  const lines = content.split("\n").filter(l => l.trim());
  const messages = [];
  for (const line of lines) {
    try { messages.push(JSON.parse(line)); } catch { /* skip */ }
  }
  return messages;
}

const TOOL_BLOCK_MAX_CHARS = 4096;

function truncateForLog(value) {
  let s;
  if (typeof value === "string") {
    s = value;
  } else {
    try {
      s = JSON.stringify(value, null, 2);
    } catch {
      s = String(value);
    }
  }
  if (typeof s !== "string") s = "";
  if (s.length <= TOOL_BLOCK_MAX_CHARS) return s;
  return (
    s.slice(0, TOOL_BLOCK_MAX_CHARS) +
    `\n... [truncated, ${s.length - TOOL_BLOCK_MAX_CHARS} more chars]`
  );
}

function extractToolResultText(content) {
  if (typeof content === "string") return content;
  if (!Array.isArray(content)) return "";
  return content
    .filter((b) => b && b.type === "text" && typeof b.text === "string")
    .map((b) => b.text)
    .join("\n");
}

function extractAllTurns(messages) {
  const turns = [];
  for (const msg of messages) {
    if (!msg || typeof msg !== "object") continue;

    let role = msg.role;
    let text = "";
    let toolNames = [];

    const harvestContent = (content) => {
      if (typeof content === "string") {
        text = content;
      } else if (Array.isArray(content)) {
        const parts = [];
        for (const block of content) {
          if (!block || typeof block !== "object") continue;
          if (block.type === "text" && typeof block.text === "string") {
            parts.push(block.text);
          } else if (block.type === "tool_use" && typeof block.name === "string") {
            toolNames.push(block.name);
            parts.push(`[tool: ${block.name}]\n${truncateForLog(block.input)}`);
          } else if (block.type === "tool_result") {
            const resultText = extractToolResultText(block.content);
            if (resultText) {
              parts.push(`[tool result]\n${truncateForLog(resultText)}`);
            }
          }
        }
        text = parts.join("\n\n");
      }
    };

    if (msg.content !== undefined) {
      harvestContent(msg.content);
    } else if (typeof msg.message === "object" && msg.message) {
      role = msg.message.role || role;
      harvestContent(msg.message.content);
    }

    if (role !== "user" && role !== "assistant") continue;
    if (!text.trim() && toolNames.length === 0) continue;
    turns.push({ role, text: text.trim(), toolNames });
  }
  return turns;
}

function formatTurnsAsText(turns) {
  const lines = [];
  for (const t of turns) {
    if (t.role === "assistant" && t.toolNames.length > 0) {
      const uniq = Array.from(new Set(t.toolNames)).join(", ");
      if (t.text) lines.push(`[assistant]: ${t.text}`);
      lines.push(`[assistant used tools: ${uniq}]`);
    } else if (t.text) {
      lines.push(`[${t.role}]: ${t.text}`);
    }
  }
  return lines.join("\n");
}

// ---------------------------------------------------------------------------
// Persistent-session capture (LoCoMo-aware created_at injection)
// ---------------------------------------------------------------------------

async function pushTurnsToOv(ovSessionId, turns, baseDate) {
  let ok = 0;
  let failed = 0;
  let idx = 0;
  let bridgesSkipped = 0;
  for (const turn of turns) {
    const content = stripInjectedBlocks(turn.text).trim();
    if (!content) continue;
    if (isCcResumeBridge(content)) { bridgesSkipped++; continue; }

    const payload = { role: turn.role, content };
    if (baseDate) {
      const ts = new Date(baseDate.getTime() + idx * 1000).toISOString();
      payload.created_at = ts;
    }

    const res = await addMessage(fetchJSON, ovSessionId, payload);
    if (res.ok) { ok++; idx++; }
    else failed++;
  }
  return { ok, failed, bridgesSkipped };
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

async function main() {
  if (!cfg.autoCapture) {
    log("skip", { stage: "init", reason: "autoCapture disabled" });
    approve();
    return;
  }

  if (await maybeDetach(cfg, { approve })) return;

  let input;
  try {
    input = JSON.parse(await readHookStdin());
  } catch {
    log("skip", { stage: "stdin_parse", reason: "invalid input" });
    approve();
    return;
  }

  const transcriptPath = input.transcript_path;
  const sessionId = input.session_id || "unknown";
  const cwd = input.cwd;
  const ovSessionId = sessionId !== "unknown" ? deriveOvSessionId(sessionId) : null;
  log("start", { sessionId, ovSessionId, transcriptPath });

  if (isBypassed(cfg, { sessionId, cwd })) {
    log("skip", { reason: "bypass_session_pattern" });
    approve();
    return;
  }

  const health = await fetchJSON("/health");
  if (!health.ok) {
    logError("health_check", "server unreachable or unhealthy");
    approve();
    return;
  }

  if (!transcriptPath || !ovSessionId) {
    log("skip", { stage: "input_check", reason: "no transcript_path or session_id" });
    approve();
    return;
  }

  let transcriptContent;
  try {
    transcriptContent = await readFile(transcriptPath, "utf-8");
  } catch (err) {
    logError("transcript_read", err);
    approve();
    return;
  }

  if (!transcriptContent.trim()) {
    log("skip", { stage: "transcript_read", reason: "empty transcript" });
    approve();
    return;
  }

  const messages = parseTranscript(transcriptContent);
  const allTurns = extractAllTurns(messages);
  if (allTurns.length === 0) {
    log("skip", { stage: "transcript_parse", reason: "no user/assistant turns found" });
    approve();
    return;
  }

  const state = await loadState(sessionId);
  const newTurns = allTurns.slice(state.capturedTurnCount);
  const captureTurns = cfg.captureAssistantTurns
    ? newTurns
    : newTurns.filter(turn => turn.role === "user");

  // LoCoMo timestamp: scan ALL captured turns (not just newTurns) so that on
  // an incremental Stop firing, the date — which lives in the very first user
  // turn header — is still recoverable. In our ingest path each `claude -p`
  // is a fresh CC session with capturedTurnCount=0, so allTurns == newTurns
  // here in practice; the fallback exists for robustness.
  const baseDate = extractBaseDate(allTurns);

  log("transcript_parse", {
    totalTurns: allTurns.length,
    previouslyCaptured: state.capturedTurnCount,
    newTurns: newTurns.length,
    captureTurns: captureTurns.length,
    assistantTurnsSkipped: newTurns.length - captureTurns.length,
    locomoBaseDate: baseDate ? baseDate.toISOString() : null,
  });

  if (newTurns.length === 0) {
    log("skip", { stage: "incremental_check", reason: "no new turns" });
    approve();
    return;
  }

  if (captureTurns.length === 0) {
    await saveState(sessionId, { capturedTurnCount: allTurns.length });
    log("state_update", { newCapturedTurnCount: allTurns.length, reason: "assistant_only_increment" });
    approve();
    return;
  }

  const combined = formatTurnsAsText(captureTurns);
  if (!sanitize(combined)) {
    log("skip", { stage: "batch_empty" });
    await saveState(sessionId, { capturedTurnCount: allTurns.length });
    approve();
    return;
  }

  if (cfg.captureMode === "keyword") {
    const hasTrigger = captureTurns.some(
      (t) =>
        t.role === "user" &&
        MEMORY_TRIGGERS.some((re) => re.test(sanitize(t.text))),
    );
    if (!hasTrigger) {
      log("skip", { stage: "keyword_mode_no_trigger", turns: captureTurns.length });
      await saveState(sessionId, { capturedTurnCount: allTurns.length });
      approve();
      return;
    }
  }

  log("should_capture", {
    capture: true,
    reason: cfg.captureMode === "keyword" ? "keyword_trigger_matched" : "semantic",
    combinedLength: combined.length,
  });

  const result = await pushTurnsToOv(ovSessionId, captureTurns, baseDate);
  log("push_turns", {
    ovSessionId,
    ok: result.ok,
    failed: result.failed,
    locomoBaseDate: baseDate ? baseDate.toISOString() : null,
  });

  await saveState(sessionId, { capturedTurnCount: allTurns.length });
  log("state_update", { newCapturedTurnCount: allTurns.length });

  let committed = false;
  let pendingTokens = 0;
  let commitCount = 0;
  let totalMessageCount = 0;
  if (result.ok > 0) {
    const meta = await getSession(fetchJSON, ovSessionId);
    pendingTokens = Number(meta?.pending_tokens || 0);
    commitCount = Number(meta?.commit_count || 0);
    totalMessageCount = Number(meta?.total_message_count || 0);
    log("pending_tokens", { ovSessionId, pending: pendingTokens, threshold: cfg.commitTokenThreshold });
    if (pendingTokens >= cfg.commitTokenThreshold) {
      const commitRes = await commitSession(fetchJSON, ovSessionId);
      committed = commitRes.ok;
      if (committed) commitCount += 1;
      log("commit", { ovSessionId, ok: commitRes.ok, pending: pendingTokens });
    }
  }

  writeJsonState("last-capture.json", {
    turns_captured: result.ok,
    turns_failed: result.failed,
    pending_tokens: pendingTokens,
    commit_threshold: cfg.commitTokenThreshold,
    committed,
    commit_count: commitCount,
    total_message_count: totalMessageCount,
    ov_session_id: ovSessionId,
    cc_session_id: sessionId,
    locomo_base_date: baseDate ? baseDate.toISOString() : null,
  });

  if (committed) {
    const today = new Date().toISOString().slice(0, 10);
    const prior = readJsonState("daily-stats.json") || {};
    const archives = prior.date === today ? Number(prior.archives || 0) + 1 : 1;
    writeJsonState("daily-stats.json", { date: today, archives });
  }

  if (result.ok > 0) {
    approve(
      `captured ${result.ok} turns to ov session ${ovSessionId}` +
      (committed ? " (committed)" : "") +
      (baseDate ? ` [locomo:${baseDate.toISOString().slice(0,10)}]` : ""),
    );
  } else {
    approve();
  }
}

main().catch((err) => { logError("uncaught", err); approve(); });