Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(codex): preserve session tool activity (#3579)
yufeng229300e88603
diff --git a/examples/claude-code-memory-plugin/scripts/shared/capture-utils.mjs b/examples/claude-code-memory-plugin/scripts/shared/capture-utils.mjsindex 3a1fb1fec..c28d9ffd9 100644--- a/examples/claude-code-memory-plugin/scripts/shared/capture-utils.mjs+++ b/examples/claude-code-memory-plugin/scripts/shared/capture-utils.mjs@@ -99,7 +99,6 @@ function toolName(block) {block?.toolName ||block?.function?.name ||block?.call?.name ||- block?.id ||"",);}@@ -128,7 +127,6 @@ function toolPayload(block, kind) {function toolId(block) {return oneLine(- block?.id ||block?.call_id ||block?.callId ||block?.tool_call_id ||@@ -137,6 +135,7 @@ function toolId(block) {block?.toolUseId ||block?.function_call_id ||block?.functionCallId ||+ block?.id ||"",);}diff --git a/examples/codex-memory-plugin/scripts/auto-capture.mjs b/examples/codex-memory-plugin/scripts/auto-capture.mjsindex ce4a57406..91758854a 100644--- a/examples/codex-memory-plugin/scripts/auto-capture.mjs+++ b/examples/codex-memory-plugin/scripts/auto-capture.mjs@@ -117,15 +117,6 @@ async function readTranscriptTurns(transcriptPath) {}}-function selectStopTurns(state, turns) {- const limit = cfg.captureMaxTurnsPerStop;- if (turns.length <= limit) return turns;- const skipped = turns.length - limit;- state.capturedTurnCount += skipped;- log("backlog_trimmed", { newTurns: turns.length, skipped, selected: limit });- return turns.slice(-limit);-}-async function appendTurns(ovSessionId, turns, state) {const payloads = turns.map((turn) => {const body = turn.parts?.length@@ -247,9 +238,7 @@ async function main() {if (!ovSessionId) {logError("resolve_ov_session", "failed to derive OV session id");} else {- const turnsToAppend = selectStopTurns(state, newTurns);- await saveState(state);- added = await appendTurns(ovSessionId, turnsToAppend, state);+ added = await appendTurns(ovSessionId, newTurns, state);log("appended", { ovSessionId, added });commitInfo = await maybeCommitByThreshold(ovSessionId, added);}diff --git a/examples/codex-memory-plugin/scripts/auto-capture.test.mjs b/examples/codex-memory-plugin/scripts/auto-capture.test.mjsindex ec464fecc..773e08632 100644--- a/examples/codex-memory-plugin/scripts/auto-capture.test.mjs+++ b/examples/codex-memory-plugin/scripts/auto-capture.test.mjs@@ -201,3 +201,103 @@ test("auto-capture commits when pending tokens cross threshold", async () => {await rm(stateDir, { recursive: true, force: true });}});++test("auto-capture sends every new turn when one response exceeds the old limit", async () => {+ const stateDir = await mkdtemp(join(tmpdir(), "ov-auto-capture-complete-"));+ const transcriptPath = join(stateDir, "transcript.jsonl");+ const calls = [];+ const transcript = [+ {+ payload: {+ message: {+ role: "user",+ content: "inspect every tool result",+ },+ },+ },+ {+ payload: {+ message: {+ role: "assistant",+ content: "I will inspect the complete trace.",+ },+ },+ },+ ];+ for (let index = 0; index < 10; index += 1) {+ transcript.push(+ {+ type: "response_item",+ payload: {+ type: "custom_tool_call",+ id: `ctc-item-${index}`,+ call_id: `custom-call-${index}`,+ status: "completed",+ name: "exec",+ input: `command-${index}`,+ },+ },+ {+ type: "response_item",+ payload: {+ type: "custom_tool_call_output",+ id: `ctco-item-${index}`,+ call_id: `custom-call-${index}`,+ output: `result-${index}`,+ },+ },+ );+ }++ try {+ await writeFile(+ transcriptPath,+ transcript.map((entry) => JSON.stringify(entry)).join("\n"),+ );++ await withMockOpenViking(async (req, res) => {+ const url = new URL(req.url, "http://127.0.0.1");+ if (req.method === "GET" && url.pathname === "/health") {+ writeJson(res, { status: "ok", result: { ok: true } });+ return;+ }+ if (req.method === "POST" && url.pathname.endsWith("/messages/batch")) {+ calls.push(await readRequestBody(req));+ writeJson(res, { status: "ok", result: { ok: true } });+ return;+ }+ if (req.method === "GET" && url.pathname === "/api/v1/sessions/cx-complete_trace") {+ writeJson(res, {+ status: "ok",+ result: { pending_tokens: 100, commit_count: 0, total_message_count: 22 },+ });+ return;+ }+ res.writeHead(404, { "Content-Type": "application/json" });+ res.end(JSON.stringify({ status: "error", error: "not found" }));+ }, async (baseUrl) => {+ await runAutoCapture(+ { session_id: "complete:trace", transcript_path: transcriptPath },+ {+ OPENVIKING_AUTO_CAPTURE: "1",+ OPENVIKING_CAPTURE_ASSISTANT_TURNS: "1",+ OPENVIKING_CODEX_STATE_DIR: stateDir,+ OPENVIKING_CONFIG_FILE: join(stateDir, "missing-ov.conf"),+ OPENVIKING_CLI_CONFIG_FILE: join(stateDir, "missing-ovcli.conf"),+ OPENVIKING_CREDENTIAL_SOURCE: "env",+ OPENVIKING_WRITE_PATH_ASYNC: "0",+ OPENVIKING_TIMEOUT_MS: "5000",+ OPENVIKING_URL: baseUrl,+ },+ );+ });++ assert.equal(+ calls.flatMap((body) => body.messages || []).length,+ 22,+ "every normalized text/tool turn must be sent",+ );+ } finally {+ await rm(stateDir, { recursive: true, force: true });+ }+});diff --git a/examples/codex-memory-plugin/scripts/capture-utils.mjs b/examples/codex-memory-plugin/scripts/capture-utils.mjsindex c681420a3..288f729ca 100644--- a/examples/codex-memory-plugin/scripts/capture-utils.mjs+++ b/examples/codex-memory-plugin/scripts/capture-utils.mjs@@ -20,6 +20,124 @@ function mcpResultText(result) {return error == null ? "" : JSON.stringify(error);}+function deduplicateMcpFunctionEvents(rolloutEntries) {+ const mcpCallIds = new Set(+ (rolloutEntries || [])+ .filter((entry) => entry?.payload?.type === "mcp_tool_call_end")+ .map((entry) => entry.payload.call_id)+ .filter(Boolean),+ );+ if (mcpCallIds.size === 0) return rolloutEntries || [];++ const duplicatedTypes = new Set([+ "function_call",+ "function_call_output",+ "custom_tool_call",+ "custom_tool_call_output",+ ]);+ return (rolloutEntries || []).filter((entry) => {+ const payload = entry?.payload;+ return !(+ duplicatedTypes.has(payload?.type)+ && mcpCallIds.has(payload.call_id || payload.id)+ );+ });+}++function agentMessageText(payload) {+ const content = Array.isArray(payload?.content) ? payload.content : [];+ return content+ .filter((part) => (+ (part?.type === "input_text" || part?.type === "output_text" || part?.type === "text")+ && typeof part.text === "string"+ ))+ .map((part) => part.text)+ .join("\n")+ .trim();+}++function normalizeCodexNativeToolEvents(rolloutEntries) {+ return (rolloutEntries || []).map((entry) => {+ const payload = entry?.payload;+ if (entry?.type === "response_item" && payload?.type === "agent_message") {+ const text = agentMessageText(payload);+ const author = String(payload.author || "subagent");+ const recipient = String(payload.recipient || "main");+ return {+ ...entry,+ payload: {+ type: "message",+ role: "assistant",+ content: [{+ type: "output_text",+ text: `[agent-message ${author} -> ${recipient}]${text ? `\n${text}` : ""}`,+ }],+ },+ };+ }+ if (payload?.type === "sub_agent_activity") {+ return {+ ...entry,+ payload: {+ type: "tool_result",+ call_id: `subagent-activity:${payload.event_id}`,+ name: "sub_agent_activity",+ output: {+ agent_thread_id: payload.agent_thread_id,+ agent_path: payload.agent_path,+ kind: payload.kind,+ occurred_at_ms: payload.occurred_at_ms,+ },+ status: "completed",+ },+ };+ }+ if (payload?.type === "custom_tool_call") {+ return {+ ...entry,+ payload: {+ type: "function_call",+ call_id: payload.call_id || payload.id,+ name: payload.name,+ arguments: payload.input,+ },+ };+ }+ if (payload?.type === "custom_tool_call_output") {+ return {+ ...entry,+ payload: {+ type: "function_call_output",+ call_id: payload.call_id || payload.id,+ output: payload.output,+ },+ };+ }+ if (payload?.type === "tool_search_call") {+ return {+ ...entry,+ payload: {+ type: "function_call",+ call_id: payload.call_id || payload.id,+ name: "tool_search",+ arguments: payload.arguments || {},+ },+ };+ }+ if (payload?.type === "tool_search_output") {+ return {+ ...entry,+ payload: {+ type: "function_call_output",+ call_id: payload.call_id || payload.id,+ output: JSON.stringify({ tools: payload.tools || [] }),+ },+ };+ }+ return entry;+ });+}+function normalizeCodexMcpToolEvents(rolloutEntries) {return (rolloutEntries || []).flatMap((entry) => {const payload = entry?.payload;@@ -91,7 +209,9 @@ function collectExperienceUsageMetadata(rolloutEntries) {export function extractCaptureTurns(rolloutEntries, cfg = {}) {const usageMetadata = collectExperienceUsageMetadata(rolloutEntries);- const turns = extractSharedCaptureTurns(normalizeCodexMcpToolEvents(rolloutEntries), cfg);+ const deduplicated = deduplicateMcpFunctionEvents(rolloutEntries);+ const normalized = normalizeCodexNativeToolEvents(deduplicated);+ const turns = extractSharedCaptureTurns(normalizeCodexMcpToolEvents(normalized), cfg);return turns.map((turn) => ({...turn,parts: turn.parts.map((part) => {diff --git a/examples/codex-memory-plugin/scripts/capture-utils.test.mjs b/examples/codex-memory-plugin/scripts/capture-utils.test.mjsindex 2d7b81c46..8412898f5 100644--- a/examples/codex-memory-plugin/scripts/capture-utils.test.mjs+++ b/examples/codex-memory-plugin/scripts/capture-utils.test.mjs@@ -3,6 +3,264 @@ import test from "node:test";import { extractCaptureTurns } from "./capture-utils.mjs";+const CAPTURE_CONFIG = {+ captureAssistantTurns: true,+ captureToolMaxChars: 2000,+ captureMaxLength: 24000,+};++test("pairs current Codex function_call records by call_id", () => {+ const turns = extractCaptureTurns(+ [+ {+ type: "response_item",+ payload: {+ type: "function_call",+ id: "fc-item-1",+ call_id: "logical-call-1",+ name: "exec_command",+ arguments: JSON.stringify({ cmd: "pwd" }),+ },+ },+ {+ type: "response_item",+ payload: {+ type: "function_call_output",+ id: "fco-item-1",+ call_id: "logical-call-1",+ output: "project root",+ },+ },+ ],+ CAPTURE_CONFIG,+ );++ assert.deepEqual(+ turns.flatMap((turn) => turn.parts),+ [+ {+ type: "tool",+ tool_id: "logical-call-1",+ tool_name: "exec_command",+ tool_status: "running",+ tool_input: { cmd: "pwd" },+ },+ {+ type: "tool",+ tool_id: "logical-call-1",+ tool_name: "exec_command",+ tool_status: "completed",+ tool_output: "project root",+ },+ ],+ );+});++test("captures current Codex custom_tool_call records", () => {+ const turns = extractCaptureTurns(+ [+ {+ type: "response_item",+ payload: {+ type: "custom_tool_call",+ id: "ctc-item-1",+ call_id: "custom-call-1",+ status: "completed",+ name: "exec",+ input: "sed -n '1,200p' /tmp/skills/tdd/SKILL.md",+ },+ },+ {+ type: "response_item",+ payload: {+ type: "custom_tool_call_output",+ id: "ctco-item-1",+ call_id: "custom-call-1",+ output: "skill contents",+ },+ },+ ],+ CAPTURE_CONFIG,+ );++ assert.deepEqual(+ turns.flatMap((turn) => turn.parts),+ [+ {+ type: "tool",+ tool_id: "custom-call-1",+ tool_name: "exec",+ tool_status: "running",+ tool_input: { value: "sed -n '1,200p' /tmp/skills/tdd/SKILL.md" },+ },+ {+ type: "tool",+ tool_id: "custom-call-1",+ tool_name: "exec",+ tool_status: "completed",+ tool_output: "skill contents",+ },+ ],+ );+});++test("captures current Codex tool_search records", () => {+ const turns = extractCaptureTurns(+ [+ {+ type: "response_item",+ payload: {+ type: "tool_search_call",+ id: "ts-item-1",+ call_id: "tool-search-1",+ status: "completed",+ arguments: { query: "calendar tools", limit: 3 },+ },+ },+ {+ type: "response_item",+ payload: {+ type: "tool_search_output",+ id: "tso-item-1",+ call_id: "tool-search-1",+ status: "completed",+ tools: [{ name: "calendar.search" }],+ },+ },+ ],+ CAPTURE_CONFIG,+ );++ assert.deepEqual(+ turns.flatMap((turn) => turn.parts),+ [+ {+ type: "tool",+ tool_id: "tool-search-1",+ tool_name: "tool_search",+ tool_status: "running",+ tool_input: { query: "calendar tools", limit: 3 },+ },+ {+ type: "tool",+ tool_id: "tool-search-1",+ tool_name: "tool_search",+ tool_status: "completed",+ tool_output: JSON.stringify({ tools: [{ name: "calendar.search" }] }),+ },+ ],+ );+});++test("deduplicates function records also reported as mcp_tool_call_end", () => {+ const turns = extractCaptureTurns(+ [+ {+ type: "response_item",+ payload: {+ type: "function_call",+ id: "fc-mcp-1",+ call_id: "mcp-call-1",+ name: "js",+ arguments: JSON.stringify({ code: "return 1" }),+ },+ },+ {+ type: "response_item",+ payload: {+ type: "function_call_output",+ id: "fco-mcp-1",+ call_id: "mcp-call-1",+ output: "1",+ },+ },+ {+ type: "event_msg",+ payload: {+ type: "mcp_tool_call_end",+ call_id: "mcp-call-1",+ invocation: {+ server: "node-repl",+ tool: "js",+ arguments: { code: "return 1" },+ },+ result: {+ Ok: {+ content: [{ type: "text", text: "1" }],+ },+ },+ },+ },+ ],+ CAPTURE_CONFIG,+ );++ const parts = turns.flatMap((turn) => turn.parts);+ assert.equal(parts.length, 2);+ assert.deepEqual(parts.map((part) => part.tool_status), ["running", "completed"]);+ assert.deepEqual(parts.map((part) => part.tool_id), ["mcp-call-1", "mcp-call-1"]);+});++test("captures Codex subagent messages and activity with identity", () => {+ const turns = extractCaptureTurns(+ [+ {+ type: "event_msg",+ payload: {+ type: "agent_message",+ message: "UI duplicate of a normal assistant response",+ },+ },+ {+ type: "response_item",+ payload: {+ type: "agent_message",+ author: "reviewer",+ recipient: "main",+ content: [{ type: "input_text", text: "Found a pairing bug." }],+ },+ },+ {+ type: "event_msg",+ payload: {+ type: "sub_agent_activity",+ event_id: "subagent-event-1",+ occurred_at_ms: 1234,+ agent_thread_id: "thread-1",+ agent_path: "reviewer",+ kind: "started",+ },+ },+ ],+ CAPTURE_CONFIG,+ );++ assert.deepEqual(turns[0], {+ role: "assistant",+ text: "[agent-message reviewer -> main]\nFound a pairing bug.",+ parts: [+ {+ type: "text",+ text: "[agent-message reviewer -> main]\nFound a pairing bug.",+ },+ ],+ });+ assert.deepEqual(turns[1].parts, [+ {+ type: "tool",+ tool_id: "subagent-activity:subagent-event-1",+ tool_name: "sub_agent_activity",+ tool_status: "completed",+ tool_output: JSON.stringify({+ agent_thread_id: "thread-1",+ agent_path: "reviewer",+ kind: "started",+ occurred_at_ms: 1234,+ }),+ },+ ]);+});+test("captures Codex mcp_tool_call_end as standard tool parts", () => {const results = {results: [@@ -32,11 +290,7 @@ test("captures Codex mcp_tool_call_end as standard tool parts", () => {},},],- {- captureAssistantTurns: true,- captureToolMaxChars: 2000,- captureMaxLength: 24000,- },+ CAPTURE_CONFIG,);assert.deepEqual(turns, [@@ -93,11 +347,7 @@ test("keeps MCP tool-level errors out of completed Experience tool parts", () =>},},],- {- captureAssistantTurns: true,- captureToolMaxChars: 2000,- captureMaxLength: 24000,- },+ CAPTURE_CONFIG,);const parts = turns.flatMap((turn) => turn.parts);@@ -132,11 +382,7 @@ test("preserves Experience usage metadata when Codex truncates a long MCP result},},],- {- captureAssistantTurns: true,- captureToolMaxChars: 2000,- captureMaxLength: 24000,- },+ CAPTURE_CONFIG,);const completed = turns@@ -174,11 +420,7 @@ test("keeps search Experience URIs parseable when snippets exceed the capture li},},],- {- captureAssistantTurns: true,- captureToolMaxChars: 2000,- captureMaxLength: 24000,- },+ CAPTURE_CONFIG,);const completed = turnsdiff --git a/examples/codex-memory-plugin/scripts/config.mjs b/examples/codex-memory-plugin/scripts/config.mjsindex 34d4f94c3..d836a9de2 100644--- a/examples/codex-memory-plugin/scripts/config.mjs+++ b/examples/codex-memory-plugin/scripts/config.mjs@@ -197,10 +197,6 @@ export function loadConfig() {process.env.OPENVIKING_CAPTURE_MAX_LENGTH,num(cx.captureMaxLength, 24000),))),- captureMaxTurnsPerStop: Math.max(1, Math.floor(num(- process.env.OPENVIKING_CAPTURE_MAX_TURNS_PER_STOP,- num(cx.captureMaxTurnsPerStop, 8),- ))),captureTimeoutMs,captureToolMaxChars: Math.max(200, Math.floor(num(process.env.OPENVIKING_CAPTURE_TOOL_MAX_CHARS,diff --git a/examples/codex-memory-plugin/scripts/shared/capture-utils.mjs b/examples/codex-memory-plugin/scripts/shared/capture-utils.mjsindex 3a1fb1fec..c28d9ffd9 100644--- a/examples/codex-memory-plugin/scripts/shared/capture-utils.mjs+++ b/examples/codex-memory-plugin/scripts/shared/capture-utils.mjs@@ -99,7 +99,6 @@ function toolName(block) {block?.toolName ||block?.function?.name ||block?.call?.name ||- block?.id ||"",);}@@ -128,7 +127,6 @@ function toolPayload(block, kind) {function toolId(block) {return oneLine(- block?.id ||block?.call_id ||block?.callId ||block?.tool_call_id ||@@ -137,6 +135,7 @@ function toolId(block) {block?.toolUseId ||block?.function_call_id ||block?.functionCallId ||+ block?.id ||"",);}diff --git a/examples/memory-plugin-shared/lib/capture-utils.mjs b/examples/memory-plugin-shared/lib/capture-utils.mjsindex 0eae451ed..4c951bc5a 100644--- a/examples/memory-plugin-shared/lib/capture-utils.mjs+++ b/examples/memory-plugin-shared/lib/capture-utils.mjs@@ -98,7 +98,6 @@ function toolName(block) {block?.toolName ||block?.function?.name ||block?.call?.name ||- block?.id ||"",);}@@ -127,7 +126,6 @@ function toolPayload(block, kind) {function toolId(block) {return oneLine(- block?.id ||block?.call_id ||block?.callId ||block?.tool_call_id ||@@ -136,6 +134,7 @@ function toolId(block) {block?.toolUseId ||block?.function_call_id ||block?.functionCallId ||+ block?.id ||"",);}diff --git a/examples/opencode-plugin/lib/shared/capture-utils.mjs b/examples/opencode-plugin/lib/shared/capture-utils.mjsindex 3a1fb1fec..c28d9ffd9 100644--- a/examples/opencode-plugin/lib/shared/capture-utils.mjs+++ b/examples/opencode-plugin/lib/shared/capture-utils.mjs@@ -99,7 +99,6 @@ function toolName(block) {block?.toolName ||block?.function?.name ||block?.call?.name ||- block?.id ||"",);}@@ -128,7 +127,6 @@ function toolPayload(block, kind) {function toolId(block) {return oneLine(- block?.id ||block?.call_id ||block?.callId ||block?.tool_call_id ||@@ -137,6 +135,7 @@ function toolId(block) {block?.toolUseId ||block?.function_call_id ||block?.functionCallId ||+ block?.id ||"",);}diff --git a/examples/pi-coding-agent-extension/shared/capture-utils.mjs b/examples/pi-coding-agent-extension/shared/capture-utils.mjsindex 3a1fb1fec..c28d9ffd9 100644--- a/examples/pi-coding-agent-extension/shared/capture-utils.mjs+++ b/examples/pi-coding-agent-extension/shared/capture-utils.mjs@@ -99,7 +99,6 @@ function toolName(block) {block?.toolName ||block?.function?.name ||block?.call?.name ||- block?.id ||"",);}@@ -128,7 +127,6 @@ function toolPayload(block, kind) {function toolId(block) {return oneLine(- block?.id ||block?.call_id ||block?.callId ||block?.tool_call_id ||@@ -137,6 +135,7 @@ function toolId(block) {block?.toolUseId ||block?.function_call_id ||block?.functionCallId ||+ block?.id ||"",);}