Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(langchain): preserve recall prompt cache prefixes (#6466)

z1gonda655e1637f3
diff --git a/docs/en/agent-integrations/07-langchain-langgraph.md b/docs/en/agent-integrations/07-langchain-langgraph.md
index aa726c183..106145e2e 100644
--- a/docs/en/agent-integrations/07-langchain-langgraph.md
+++ b/docs/en/agent-integrations/07-langchain-langgraph.md
@@ -288,6 +288,19 @@ middleware = OpenVikingContextMiddleware(
)
```
+The middleware prepends recalled context to a copy of the current user message
+for the model request. System instructions and stored conversation messages
+remain unchanged. Recall is reused within tool loops and refreshed for each new
+user message.
+
+With a LangGraph checkpointer, the middleware saves exact recall blocks in
+internal graph state and replays them with their original messages after later
+turns or a restart. This preserves unchanged prompt prefixes for provider caching;
+actual cache hits depend on the provider. The ledger retains up to 500 entries
+per session/peer scope and follows the application's checkpoint retention policy.
+Pending structured context references are also checkpointed until their assistant
+message is captured, so resuming an interrupted tool loop preserves provenance.
+
### Session recorder
Use the recorder when your application already owns the conversation lifecycle
diff --git a/docs/zh/agent-integrations/07-langchain-langgraph.md b/docs/zh/agent-integrations/07-langchain-langgraph.md
index 6281d8a37..835c12683 100644
--- a/docs/zh/agent-integrations/07-langchain-langgraph.md
+++ b/docs/zh/agent-integrations/07-langchain-langgraph.md
@@ -275,6 +275,15 @@ middleware = OpenVikingContextMiddleware(
)
```
+中间件会将召回上下文添加到当前用户消息的副本中,再发送给模型;系统指令和已保存的
+对话消息保持不变。同一轮工具调用复用召回结果,新的用户消息会重新召回。
+
+使用 LangGraph checkpointer 时,中间件会在内部图状态中保存完整召回文本,供后续轮次或
+重启后随原消息重放,从而保留一致的提示词前缀,便于模型服务复用缓存。实际缓存命中取决于
+模型服务。每个 session/peer 范围最多保留 500 条记录,并遵循应用的 checkpoint 保留策略。
+尚未写入 OpenViking 的结构化上下文引用也会保存到 checkpoint,直到对应的助手消息被确认写入,
+因此工具调用中断后恢复时仍能保留召回来源。
+
### Session recorder
当应用已经自行管理会话生命周期,只需要复用 OpenViking 持久化能力时,可使用 recorder:
diff --git a/integrations/frameworks/langchain/examples/langgraph/middleware/quick_app.py b/integrations/frameworks/langchain/examples/langgraph/middleware/quick_app.py
index b78d6e57b..afa2e5404 100644
--- a/integrations/frameworks/langchain/examples/langgraph/middleware/quick_app.py
+++ b/integrations/frameworks/langchain/examples/langgraph/middleware/quick_app.py
@@ -56,11 +56,10 @@ def build_app(client: InMemoryOpenVikingClient | None = None):
return new_request
def handler(request):
- context = (
- extract_message_text(request.system_message.content)
- if request.system_message
- else ""
- )
+ model_messages = list(request.messages)
+ if request.system_message is not None:
+ model_messages.insert(0, request.system_message)
+ context = "\n".join(extract_message_text(message.content) for message in model_messages)
if "azure" in context.lower():
return AIMessage(content="OpenViking middleware context says azure.")
return AIMessage(content="OpenViking middleware context was missing.")
diff --git a/integrations/frameworks/langchain/src/langchain_openviking/middleware.py b/integrations/frameworks/langchain/src/langchain_openviking/middleware.py
index d61c4c7de..99bbb6f53 100644
--- a/integrations/frameworks/langchain/src/langchain_openviking/middleware.py
+++ b/integrations/frameworks/langchain/src/langchain_openviking/middleware.py
@@ -8,17 +8,17 @@ import asyncio
import hashlib
import json
from contextlib import AbstractContextManager, nullcontext
+from copy import deepcopy
from dataclasses import dataclass
-from typing import Any, Callable
+from typing import Annotated, Any, Callable, TypedDict
try:
from langchain.agents.middleware import AgentMiddleware
- from langchain.agents.middleware.types import AgentState, ModelRequest
+ from langchain.agents.middleware.types import AgentState, ModelRequest, PrivateStateAttr
from langchain_core.messages import (
AIMessage,
BaseMessage,
HumanMessage,
- SystemMessage,
ToolMessage,
)
from langgraph.config import get_config
@@ -36,6 +36,7 @@ from langchain_openviking.client import (
OpenVikingCommitPolicy,
extract_message_text,
get_latest_user_text,
+ get_message_role,
)
from langchain_openviking.context import (
OpenVikingAssembledContext,
@@ -54,6 +55,7 @@ _SESSION_ID_ERROR = (
'config={"configurable": {"thread_id": "..."}}, set state["session_id"], '
"or provide session_id_resolver."
)
+_RECALL_LEDGER_MAX_ENTRIES = 500
_CaptureKey = tuple[str, str] | tuple[str, str, str]
_ActorPeerResolver = Callable[
[dict[str, Any], Any],
@@ -74,10 +76,20 @@ class _CapturePlan:
unchanged: bool = False
+class _PendingContext(TypedDict):
+ delivery_id: str
+ parts: list[dict[str, Any]]
+
+
class OpenVikingCaptureState(AgentState):
# Persist delivery IDs in the host checkpointer, independently of message
# retention, so a compacted tail is not recorded again after a restart.
openviking_captured_message_ids: dict[str, list[str]]
+ # Recall is request-only, so keep the exact block separately and replay it
+ # onto the same user message when the host rebuilds model-bound history.
+ openviking_recall_ledger: Annotated[dict[str, dict[str, str]], PrivateStateAttr]
+ # References are pending only until their assistant message is acknowledged.
+ openviking_pending_context: Annotated[dict[str, _PendingContext], PrivateStateAttr]
class OpenVikingContextMiddleware(AgentMiddleware[OpenVikingCaptureState]):
@@ -181,6 +193,7 @@ class OpenVikingContextMiddleware(AgentMiddleware[OpenVikingCaptureState]):
self._captured_signatures: dict[_CaptureKey, tuple[str, ...]] = {}
self._captured_message_ids: dict[_CaptureKey, set[str]] = {}
self._pending_context_parts: dict[_CaptureKey, list[dict[str, Any]]] = {}
+ self._pending_recall_blocks: dict[_CaptureKey, tuple[str, str]] = {}
async def aclose(self) -> None:
"""Release all clients internally owned by this middleware."""
@@ -203,18 +216,20 @@ class OpenVikingContextMiddleware(AgentMiddleware[OpenVikingCaptureState]):
if plan is None:
return handler(request)
session_id, pending_key, query, actor_peer_id = plan
- with self._actor_peer_scope(actor_peer_id):
- assembled = self.assembler.assemble(
- session_id=session_id,
- query=query,
- )
- updated_request = self._request_with_context(request, pending_key, assembled)
- if updated_request is None:
- return handler(request)
+ if self._has_current_recall(request, pending_key):
+ updated_request = self._request_with_replayed_context(request, pending_key)
+ else:
+ self._discard_pending_recall(pending_key)
+ with self._actor_peer_scope(actor_peer_id):
+ assembled = self.assembler.assemble(
+ session_id=session_id,
+ query=query,
+ )
+ updated_request = self._request_with_context(request, pending_key, assembled)
try:
- return handler(updated_request)
- except Exception:
- self._pending_context_parts.pop(pending_key, None)
+ return handler(updated_request or request)
+ except BaseException:
+ self._discard_pending_recall(pending_key)
raise
async def awrap_model_call(
@@ -228,20 +243,22 @@ class OpenVikingContextMiddleware(AgentMiddleware[OpenVikingCaptureState]):
if plan is None:
return await handler(request)
session_id, pending_key, query, actor_peer_id = plan
- with self._actor_peer_scope(actor_peer_id):
- assembled = await self.assembler.aassemble(
- session_id=session_id,
- query=query,
- )
- updated_request = self._request_with_context(request, pending_key, assembled)
- if updated_request is None:
- return await handler(request)
+ if self._has_current_recall(request, pending_key):
+ updated_request = self._request_with_replayed_context(request, pending_key)
+ else:
+ self._discard_pending_recall(pending_key)
+ with self._actor_peer_scope(actor_peer_id):
+ assembled = await self.assembler.aassemble(
+ session_id=session_id,
+ query=query,
+ )
+ updated_request = self._request_with_context(request, pending_key, assembled)
try:
- return await handler(updated_request)
+ return await handler(updated_request or request)
except BaseException:
# CancelledError is a BaseException, so cancellation must also discard
# context references prepared for the interrupted model call.
- self._pending_context_parts.pop(pending_key, None)
+ self._discard_pending_recall(pending_key)
raise
def _model_context_plan(
@@ -257,9 +274,12 @@ class OpenVikingContextMiddleware(AgentMiddleware[OpenVikingCaptureState]):
peer_id = self._resolve_peer_id(state, runtime)
actor_peer_id = self._resolve_actor_peer_id(state, runtime)
pending_key = _capture_key(session_id, peer_id, actor_peer_id)
- self._pending_context_parts.pop(pending_key, None)
return session_id, pending_key, query, actor_peer_id
+ def _discard_pending_recall(self, pending_key: _CaptureKey) -> None:
+ self._pending_context_parts.pop(pending_key, None)
+ self._pending_recall_blocks.pop(pending_key, None)
+
def _request_with_context(
self,
request: ModelRequest,
@@ -267,34 +287,137 @@ class OpenVikingContextMiddleware(AgentMiddleware[OpenVikingCaptureState]):
assembled: OpenVikingAssembledContext,
) -> ModelRequest | None:
context_block = assembled.block
- if not context_block:
- return None
if assembled.context_parts:
self._pending_context_parts[pending_key] = assembled.context_parts
+ message_key = _latest_user_recall_key(request.messages)
+ if message_key is not None:
+ self._pending_recall_blocks[pending_key] = (message_key, context_block)
+ return self._request_with_replayed_context(request, pending_key)
+ return self._request_with_replayed_context(
+ request,
+ pending_key,
+ untracked_current_block=context_block,
+ )
- system_message = request.system_message
- if system_message is None:
- updated_system = SystemMessage(content=context_block)
- else:
- content = extract_message_text(system_message.content)
- updated_system = SystemMessage(content=f"{content}\n\n{context_block}".strip())
- return request.override(system_message=updated_system)
+ def _has_current_recall(self, request: ModelRequest, pending_key: _CaptureKey) -> bool:
+ message_key = _latest_user_recall_key(request.messages)
+ if message_key is None:
+ return False
+ ledger = _recall_ledger(getattr(request, "state", {}) or {}, pending_key)
+ if message_key in ledger:
+ return True
+ pending = self._pending_recall_blocks.get(pending_key)
+ return pending is not None and pending[0] == message_key
+
+ def _request_with_replayed_context(
+ self,
+ request: ModelRequest,
+ pending_key: _CaptureKey,
+ *,
+ untracked_current_block: str | None = None,
+ ) -> ModelRequest | None:
+ messages = list(request.messages)
+ latest_user_index = _latest_user_index(messages)
+ if latest_user_index is None:
+ return None
+ ledger = _recall_ledger(getattr(request, "state", {}) or {}, pending_key)
+ pending = self._pending_recall_blocks.get(pending_key)
+ if pending is not None:
+ ledger[pending[0]] = pending[1]
+
+ changed = False
+ for index, message in enumerate(messages):
+ if get_message_role(message) != "user":
+ continue
+ message_key = _recall_message_key(message)
+ if message_key is not None and message_key in ledger:
+ block = ledger[message_key]
+ elif index == latest_user_index and untracked_current_block is not None:
+ block = untracked_current_block
+ else:
+ continue
+ if not block:
+ continue
+ messages[index] = _message_with_recall(message, block)
+ changed = True
+ return request.override(messages=messages) if changed else None
def before_model(self, state: OpenVikingCaptureState, runtime: Any) -> dict[str, Any] | None:
"""Place this middleware before the host summarizer to capture its input."""
return self._capture(state, runtime, commit=False)
+ def after_model(
+ self,
+ state: OpenVikingCaptureState,
+ runtime: Any,
+ ) -> dict[str, Any] | None:
+ return self._save_pending_recall(state, runtime)
+
async def abefore_model(
self, state: OpenVikingCaptureState, runtime: Any
) -> dict[str, Any] | None:
return await self._acapture(state, runtime, commit=False)
+ async def aafter_model(
+ self,
+ state: OpenVikingCaptureState,
+ runtime: Any,
+ ) -> dict[str, Any] | None:
+ return self._save_pending_recall(state, runtime)
+
+ def _save_pending_recall(
+ self,
+ state: OpenVikingCaptureState,
+ runtime: Any,
+ ) -> dict[str, Any] | None:
+ if not self._pending_recall_blocks:
+ return None
+ pending_key = _capture_key(
+ self._resolve_session_id(state, runtime),
+ self._resolve_peer_id(state, runtime),
+ self._resolve_actor_peer_id(state, runtime),
+ )
+ pending = self._pending_recall_blocks.pop(pending_key, None)
+ if pending is None:
+ return None
+ message_key, block = pending
+ ledgers = dict(state.get("openviking_recall_ledger") or {})
+ scope_key = _stable_json(pending_key)
+ ledger = dict(ledgers.get(scope_key) or {})
+ update: dict[str, Any] = {}
+ if ledger.get(message_key) != block:
+ ledger[message_key] = block
+ while len(ledger) > _RECALL_LEDGER_MAX_ENTRIES:
+ del ledger[next(iter(ledger))]
+ ledgers[scope_key] = ledger
+ update["openviking_recall_ledger"] = ledgers
+
+ parts = self._pending_context_parts.get(pending_key)
+ messages = state.get("messages") or []
+ delivery_id = _message_delivery_id(messages[-1]) if messages else None
+ if parts and delivery_id:
+ pending_context = dict(state.get("openviking_pending_context") or {})
+ pending_context[scope_key] = {
+ "delivery_id": delivery_id,
+ "parts": deepcopy(parts),
+ }
+ update["openviking_pending_context"] = pending_context
+ return update or None
+
def _capture_state_update(
self, state: OpenVikingCaptureState, plan: _CapturePlan
) -> dict[str, Any]:
delivered = dict(state.get("openviking_captured_message_ids") or {})
delivered[_stable_json(plan.key)] = sorted(self._captured_message_ids.get(plan.key, set()))
- return {"openviking_captured_message_ids": delivered}
+ update: dict[str, Any] = {"openviking_captured_message_ids": delivered}
+ pending_context = state.get("openviking_pending_context") or {}
+ scope_key = _stable_json(plan.key)
+ pending = pending_context.get(scope_key)
+ if pending and pending["delivery_id"] in self._captured_message_ids.get(plan.key, set()):
+ pending_context = dict(pending_context)
+ del pending_context[scope_key]
+ update["openviking_pending_context"] = pending_context
+ return {key: value for key, value in update.items() if state.get(key) != value}
def _remember_capture_ids(self, plan: _CapturePlan, end: int | None = None) -> None:
delivered = self._captured_message_ids.setdefault(plan.key, set())
@@ -317,7 +440,7 @@ class OpenVikingContextMiddleware(AgentMiddleware[OpenVikingCaptureState]):
with self._actor_peer_scope(plan.actor_peer_id):
self.recorder.record(plan.session_id, (), commit_policy=policy)
self._pending_context_parts.pop(plan.key, None)
- return None
+ return self._capture_state_update(state, plan) or None
try:
with self._actor_peer_scope(plan.actor_peer_id):
result = self.recorder.record(
@@ -353,7 +476,7 @@ class OpenVikingContextMiddleware(AgentMiddleware[OpenVikingCaptureState]):
with self._actor_peer_scope(plan.actor_peer_id):
await self.recorder.arecord(plan.session_id, (), commit_policy=policy)
self._pending_context_parts.pop(plan.key, None)
- return None
+ return self._capture_state_update(state, plan) or None
try:
with self._actor_peer_scope(plan.actor_peer_id):
result = await self.recorder.arecord(
@@ -409,6 +532,14 @@ class OpenVikingContextMiddleware(AgentMiddleware[OpenVikingCaptureState]):
and signatures[: len(previous_signatures)] == previous_signatures
):
start = len(previous_signatures)
+ context_parts = list(self._pending_context_parts.get(key, []))
+ if not context_parts:
+ # Restore references only for their still-uncaptured assistant message.
+ pending = (state.get("openviking_pending_context") or {}).get(_stable_json(key))
+ if pending and any(
+ _message_delivery_id(message) == pending["delivery_id"] for message in messages
+ ):
+ context_parts = list(pending["parts"])
return _CapturePlan(
session_id=session_id,
peer_id=peer_id,
@@ -417,7 +548,7 @@ class OpenVikingContextMiddleware(AgentMiddleware[OpenVikingCaptureState]):
messages=messages,
signatures=signatures,
start=start,
- context_parts=list(self._pending_context_parts.get(key, [])),
+ context_parts=context_parts,
unchanged=unchanged,
)
@@ -616,6 +747,55 @@ def _message_stable_id(message: Any) -> str | None:
return str(value) if value else None
+def _latest_user_index(messages: list[Any]) -> int | None:
+ for index in range(len(messages) - 1, -1, -1):
+ if get_message_role(messages[index]) == "user":
+ return index
+ return None
+
+
+def _recall_message_key(message: Any) -> str | None:
+ if get_message_role(message) != "user":
+ return None
+ message_id = _message_stable_id(message)
+ if message_id is None:
+ return None
+ digest = hashlib.sha256(_message_content(message).encode()).hexdigest()
+ return f"{message_id}:{digest}"
+
+
+def _latest_user_recall_key(messages: list[Any]) -> str | None:
+ index = _latest_user_index(messages)
+ return None if index is None else _recall_message_key(messages[index])
+
+
+def _recall_ledger(
+ state: dict[str, Any],
+ pending_key: _CaptureKey,
+) -> dict[str, str]:
+ ledgers = state.get("openviking_recall_ledger") or {}
+ if not isinstance(ledgers, dict):
+ return {}
+ raw = ledgers.get(_stable_json(pending_key)) or {}
+ if not isinstance(raw, dict):
+ return {}
+ return {
+ key: block for key, block in raw.items() if isinstance(key, str) and isinstance(block, str)
+ }
+
+
+def _message_with_recall(message: Any, block: str) -> Any:
+ content = message.get("content", "") if isinstance(message, dict) else message.content
+ if isinstance(content, list):
+ updated_content: Any = [{"type": "text", "text": block}, *content]
+ else:
+ text = content if isinstance(content, str) else extract_message_text(content)
+ updated_content = f"{block}\n\n{text}" if text else block
+ if isinstance(message, dict):
+ return {**message, "content": updated_content}
+ return message.model_copy(update={"content": updated_content})
+
+
def _message_delivery_id(message: Any) -> str | None:
if not _message_stable_id(message):
return None
diff --git a/integrations/frameworks/langchain/tests/unit/test_langchain_async_integration.py b/integrations/frameworks/langchain/tests/unit/test_langchain_async_integration.py
index 1d7904c54..3e5773e33 100644
--- a/integrations/frameworks/langchain/tests/unit/test_langchain_async_integration.py
+++ b/integrations/frameworks/langchain/tests/unit/test_langchain_async_integration.py
@@ -13,7 +13,7 @@ pytest.importorskip("langchain_core")
pytest.importorskip("langgraph")
pytest.importorskip("langchain_openviking")
-from langchain_core.messages import AIMessage, HumanMessage
+from langchain_core.messages import AIMessage, ChatMessage, HumanMessage
from langchain_openviking import (
InMemoryOpenVikingClient,
@@ -1486,7 +1486,8 @@ async def test_async_history_does_not_create_session_on_non_not_found_error():
@pytest.mark.asyncio
-async def test_async_middleware_injects_and_captures_context():
[email protected]("generic_user", [False, True])
+async def test_async_middleware_injects_and_captures_context(generic_user):
backing = InMemoryOpenVikingClient(
{"viking://~/memories/profile.md": "Async middleware prefers teal."}
)
@@ -1501,12 +1502,18 @@ async def test_async_middleware_injects_and_captures_context():
class Request:
state: dict[str, Any] = {}
runtime = None
- messages = [HumanMessage(content="What teal preference?")]
+ messages = [
+ ChatMessage(role="user", id="async-user-1", content="What teal preference?")
+ if generic_user
+ else HumanMessage(id="async-user-1", content="What teal preference?")
+ ]
system_message = None
def override(self, **overrides: Any) -> Request:
request = Request()
+ request.messages = overrides.get("messages", self.messages)
request.system_message = overrides.get("system_message", self.system_message)
+ request.state = self.state
return request
async def handler(request: Any) -> AIMessage:
@@ -1514,25 +1521,32 @@ async def test_async_middleware_injects_and_captures_context():
return AIMessage(content="teal")
await middleware.awrap_model_call(Request(), handler)
+ recall_update = await middleware.aafter_model({}, runtime=None)
+ assert recall_update is not None
await middleware.aafter_agent(
{
+ **recall_update,
"messages": [
HumanMessage(content="What color?"),
AIMessage(content="teal"),
- ]
+ ],
},
runtime=None,
)
- assert "Async middleware prefers teal." in (captured_request["request"].system_message.content)
+ assert captured_request["request"].system_message is None
+ assert "Async middleware prefers teal." in captured_request["request"].messages[0].content
assistant_parts = backing.sessions["async-middleware"][1]["parts"]
assert any(part["type"] == "context" for part in assistant_parts)
@pytest.mark.asyncio
-async def test_async_middleware_clears_pending_context_when_model_call_is_cancelled():
[email protected]("has_context", [False, True])
+async def test_async_middleware_clears_pending_context_when_model_call_is_cancelled(has_context):
backing = InMemoryOpenVikingClient(
{"viking://~/memories/profile.md": "Cancelled context must not be reused."}
+ if has_context
+ else {}
)
client = AsyncInMemoryOpenVikingClient(backing)
middleware = OpenVikingContextMiddleware(
@@ -1545,12 +1559,14 @@ async def test_async_middleware_clears_pending_context_when_model_call_is_cancel
class Request:
state: dict[str, Any] = {}
runtime = None
- messages = [HumanMessage(content="Find the cancelled context.")]
+ messages = [HumanMessage(id="async-cancelled-user", content="Find the cancelled context.")]
system_message = None
def override(self, **overrides: Any) -> Request:
request = Request()
+ request.messages = overrides.get("messages", self.messages)
request.system_message = overrides.get("system_message", self.system_message)
+ request.state = self.state
return request
async def handler(_request: Any) -> AIMessage:
@@ -1566,3 +1582,5 @@ async def test_async_middleware_clears_pending_context_when_model_call_is_cancel
await model_call
assert middleware._pending_context_parts == {}
+ assert middleware._pending_recall_blocks == {}
+ assert await middleware.aafter_model({"messages": Request.messages}, runtime=None) is None
diff --git a/integrations/frameworks/langchain/tests/unit/test_langchain_integration.py b/integrations/frameworks/langchain/tests/unit/test_langchain_integration.py
index a90e24fb2..fa09cfa6d 100644
--- a/integrations/frameworks/langchain/tests/unit/test_langchain_integration.py
+++ b/integrations/frameworks/langchain/tests/unit/test_langchain_integration.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import json
+import sys
from copy import deepcopy
from types import MappingProxyType
from typing import Any
@@ -2255,7 +2256,9 @@ def test_pending_token_commit_uses_persisted_value_without_session_lookup():
assert client.get_session_calls == 0
-def test_langgraph_middleware_injects_recall_and_captures_messages():
[email protected]("shape", ["human", "chat", "chat_chunk", "dict"])
[email protected]("structured", [False, True])
+def test_langgraph_middleware_injects_recall_and_captures_messages(shape, structured):
client = InMemoryOpenVikingClient(
{"viking://~/memories/profile.md": "The user prefers azure deployments."}
)
@@ -2267,13 +2270,40 @@ def test_langgraph_middleware_injects_recall_and_captures_messages():
)
captured_request = {}
+ content = "What deployment color?"
+ if structured:
+ content = [
+ {"type": "text", "text": content},
+ {"type": "image_url", "image_url": {"url": "https://example.com/deployment.png"}},
+ ]
+ values = {
+ "id": "middleware-user-1",
+ "name": "customer",
+ "content": content,
+ "additional_kwargs": {"host_field": {"keep": True}},
+ }
+ if shape == "dict":
+ user = {"role": "user", **values}
+ elif shape == "human":
+ user = HumanMessage(**values)
+ else:
+ message_type = ChatMessage if shape == "chat" else ChatMessageChunk
+ user = message_type(role="user", **values)
+ original = deepcopy(user)
+ system = SystemMessage(
+ id="policy",
+ content=[{"type": "text", "text": "Stable agent policy."}],
+ additional_kwargs={"host_field": {"keep": True}},
+ )
class Request:
- messages = [HumanMessage(content="What deployment color?")]
- system_message = None
+ def __init__(self, state=None):
+ self.state = state or {}
+ self.messages = [user]
+ self.system_message = system
def override(self, **overrides):
- new_request = Request()
+ new_request = Request(self.state)
new_request.messages = overrides.get("messages", self.messages)
new_request.system_message = overrides.get("system_message", self.system_message)
return new_request
@@ -2283,15 +2313,50 @@ def test_langgraph_middleware_injects_recall_and_captures_messages():
return AIMessage(content="ok")
middleware.wrap_model_call(Request(), handler)
- assert "OpenViking context" in captured_request["request"].system_message.content
- assert "azure deployments" in captured_request["request"].system_message.content
+ first_request = captured_request["request"]
+ recall_update = middleware.after_model({}, runtime=None)
+ assert recall_update is not None
+ assert first_request.system_message is system
+ injected = first_request.messages[0]
+ injected_values = injected if isinstance(injected, dict) else injected.model_dump()
+ original_values = original if isinstance(original, dict) else original.model_dump()
+ assert "azure deployments" in str(injected_values["content"])
+ assert {k: v for k, v in injected_values.items() if k != "content"} == {
+ k: v for k, v in original_values.items() if k != "content"
+ }
+ if structured:
+ assert injected_values["content"][1:] == content
+ assert user == original
+
+ # A later model call for the same user turn replays the recorded block,
+ # without another search or dropping context parts needed by capture.
+ middleware.wrap_model_call(Request(recall_update), handler)
+ replayed_request = captured_request["request"]
+ assert replayed_request.messages == first_request.messages
+ assert len(client.search_calls) == 1
+
+ # Editing the same message ID requires fresh recall rather than a stale replay.
+ client.records["viking://~/memories/profile.md"] = "Azure deployments must use Singapore."
+ edited_request = Request(recall_update)
+ edited_request.messages = [
+ {**user, "content": "Which azure region?"}
+ if isinstance(user, dict)
+ else user.model_copy(update={"content": "Which azure region?"})
+ ]
+ middleware.wrap_model_call(edited_request, handler)
+ assert len(client.search_calls) == 2
+ edited = captured_request["request"].messages[0]
+ edited_content = edited["content"] if isinstance(edited, dict) else edited.content
+ assert "Singapore" in edited_content
+ assert user == original
middleware.after_agent(
{
+ **recall_update,
"messages": [
HumanMessage(content="Remember this."),
AIMessage(content="I will."),
- ]
+ ],
},
runtime=None,
)
@@ -2303,6 +2368,227 @@ def test_langgraph_middleware_injects_recall_and_captures_messages():
assert any(part["type"] == "context" for part in assistant_parts)
[email protected]("asynchronous", [False, True])
[email protected]("streaming", [False, True])
[email protected]("capture", [False, True])
[email protected]("interrupt_tools", [False, True])
+async def test_langgraph_middleware_replays_exact_recall_prefix_after_restart(
+ asynchronous, streaming, capture, interrupt_tools
+):
+ from langchain.agents import create_agent
+ from langchain_core.language_models.chat_models import BaseChatModel
+ from langchain_core.messages import AIMessageChunk
+ from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
+ from langgraph.checkpoint.memory import InMemorySaver
+ from pydantic import Field
+
+ class RecordingModel(BaseChatModel):
+ requests: list[list[Any]] = Field(default_factory=list)
+ response_count: int = 0
+
+ @property
+ def _llm_type(self):
+ return "recording"
+
+ def bind_tools(self, tools, **kwargs):
+ return self
+
+ def _reply(self, messages):
+ self.requests.append(list(messages))
+ self.response_count += 1
+ if messages[-1].type == "tool":
+ return AIMessage(content=f"answer-{self.response_count}")
+ user = next(message for message in reversed(messages) if message.type == "human")
+ return AIMessage(
+ content="Checking the deployment.",
+ tool_calls=[{"id": f"lookup-{user.id}", "name": "lookup_deployment", "args": {}}],
+ )
+
+ def _generate(self, messages, stop=None, run_manager=None, **kwargs):
+ return ChatResult(generations=[ChatGeneration(message=self._reply(messages))])
+
+ def _stream(self, messages, stop=None, run_manager=None, **kwargs):
+ response = self._reply(messages)
+ yield ChatGenerationChunk(
+ message=AIMessageChunk(content=response.content, tool_calls=response.tool_calls)
+ )
+
+ async def _astream(self, messages, stop=None, run_manager=None, **kwargs):
+ for chunk in self._stream(messages, stop, run_manager, **kwargs):
+ yield chunk
+
+ def lookup_deployment() -> str:
+ """Check the current deployment status."""
+ return "The deployment is ready."
+
+ def request_bytes(messages):
+ wire = convert_to_openai_messages(messages, text_format="block")
+ return json.dumps(wire, ensure_ascii=False, separators=(",", ":")).encode()
+
+ client = InMemoryOpenVikingClient(
+ {"viking://~/memories/profile.md": "The user prefers azure deployments."}
+ )
+ checkpointer = InMemorySaver()
+ config = {"configurable": {"thread_id": "prefix-session"}}
+
+ def build_agent(model, *, pause_before_tools=False):
+ return create_agent(
+ model,
+ tools=[lookup_deployment],
+ system_prompt="Stable agent policy.",
+ middleware=[
+ OpenVikingContextMiddleware(
+ client=client,
+ target_uri="viking://~/memories",
+ capture_on_after_agent=capture,
+ # Python 3.10 async tasks cannot resolve the graph's config context.
+ session_id_resolver=lambda _state, _runtime: "prefix-session",
+ )
+ ],
+ checkpointer=checkpointer,
+ interrupt_before=["tools"] if pause_before_tools else None,
+ )
+
+ async def run(agent, text):
+ payload = None if text is None else {"messages": [{"role": "user", "content": text}]}
+ if streaming:
+ events = (
+ [
+ event
+ async for event in agent.astream(
+ payload, config, stream_mode=["messages", "updates"]
+ )
+ ]
+ if asynchronous
+ else list(agent.stream(payload, config, stream_mode=["messages", "updates"]))
+ )
+ assert any(mode == "updates" for mode, _ in events)
+ # Python 3.10 lacks implicit async callback propagation in LangChain 1.0.
+ if not asynchronous or sys.version_info >= (3, 11):
+ assert any(
+ mode == "messages" and isinstance(event[0], AIMessageChunk)
+ for mode, event in events
+ )
+ elif asynchronous:
+ await agent.ainvoke(payload, config)
+ else:
+ agent.invoke(payload, config)
+
+ model = RecordingModel()
+ agent = build_agent(model)
+ for schema in (agent.get_input_schema(), agent.get_output_schema()):
+ assert "openviking_recall_ledger" not in schema.model_json_schema().get("properties", {})
+ assert "openviking_pending_context" not in schema.model_json_schema().get("properties", {})
+ await run(agent, "first azure deployment prompt")
+ client.records["viking://~/memories/profile.md"] = "Azure deployments must use Singapore."
+ if interrupt_tools:
+ agent = build_agent(model, pause_before_tools=True)
+ await run(agent, "second azure deployment prompt")
+ state = agent.get_state(config).values
+
+ assert len(client.search_calls) == 2
+ assert all(message.id for message in state["messages"] if message.type == "human")
+ assert [message.content for message in state["messages"] if message.type == "human"] == [
+ "first azure deployment prompt",
+ "second azure deployment prompt",
+ ]
+ assert sum(len(ledger) for ledger in state["openviking_recall_ledger"].values()) == 2
+ assert len(model.requests) == (3 if interrupt_tools else 4)
+ assert "azure deployments" in model.requests[0][1].content
+ assert "Singapore" in model.requests[2][-1].content
+
+ # A fresh middleware and graph recover the ledger from the same checkpointer.
+ restarted_model = RecordingModel()
+ restarted_agent = build_agent(restarted_model)
+ if interrupt_tools:
+ assert agent.get_state(config).next == ("tools",)
+ await run(restarted_agent, None)
+ assert len(client.search_calls) == 2
+ references = [
+ part
+ for message in client.sessions["prefix-session"]
+ for part in message.get("parts", [])
+ if part["type"] == "context"
+ ]
+ assert len(references) == (2 if capture else 0)
+ if capture:
+ assert references[-1] == {
+ "type": "context",
+ "uri": "viking://~/memories/profile.md",
+ "context_type": "memory",
+ "abstract": "Azure deployments must use Singapore.",
+ }
+ await run(restarted_agent, "third azure deployment prompt")
+ restarted_state = restarted_agent.get_state(config).values
+
+ assert len(client.search_calls) == 3
+ requests = model.requests + restarted_model.requests
+ for previous, current in zip(requests, requests[1:], strict=False):
+ assert request_bytes(current[: len(previous)]) == request_bytes(previous)
+ assert current[0].content == "Stable agent policy."
+ assert sum(len(ledger) for ledger in restarted_state["openviking_recall_ledger"].values()) == 3
+ captured = client.sessions["prefix-session"]
+ if capture:
+ assert len(captured) == 12
+ parts = [part for message in captured for part in message.get("parts", [])]
+ assert not any(
+ "<openviking_context>" in part.get("text", "")
+ for part in parts
+ if part["type"] == "text"
+ )
+ assert len([part for part in parts if part["type"] == "tool"]) == 6
+ assert len([part for part in parts if part["type"] == "context"]) == 3
+ assert [
+ message.content for message in restarted_state["messages"] if message.type == "human"
+ ] == [
+ "first azure deployment prompt",
+ "second azure deployment prompt",
+ "third azure deployment prompt",
+ ]
+ assert not restarted_state.get("openviking_pending_context")
+ else:
+ assert captured == []
+
+
[email protected]("has_context", [False, True])
[email protected]("error_type", [RuntimeError, KeyboardInterrupt])
+def test_langgraph_middleware_discards_recall_when_model_call_fails(has_context, error_type):
+ client = InMemoryOpenVikingClient(
+ {"viking://~/memories/profile.md": "Azure deployments use a stale region."}
+ if has_context
+ else {}
+ )
+ middleware = OpenVikingContextMiddleware(
+ client=client,
+ session_id_resolver=lambda _state, _runtime: "empty-failure-session",
+ )
+
+ class Request:
+ state = {}
+ runtime = None
+ messages = [HumanMessage(id="empty-failure-user", content="Find azure deployments.")]
+ system_message = None
+
+ def override(self, **overrides):
+ request = Request()
+ request.messages = overrides.get("messages", self.messages)
+ return request
+
+ def handler(_request):
+ raise error_type("model failed")
+
+ with pytest.raises(error_type, match="model failed"):
+ middleware.wrap_model_call(Request(), handler)
+
+ assert middleware.after_model({"messages": Request.messages}, runtime=None) is None
+ client.records["viking://~/memories/profile.md"] = "Azure deployments now use Singapore."
+ seen = []
+ middleware.wrap_model_call(Request(), lambda request: seen.append(request))
+ assert len(client.search_calls) == 2
+ assert "Singapore" in seen[0].messages[0].content
+
+
def test_langgraph_middleware_batches_persistence_before_separate_commit():
class RecordingClient(InMemoryOpenVikingClient):
def __init__(self):
@@ -2432,10 +2718,11 @@ def test_langgraph_middleware_does_not_duplicate_active_messages_in_context():
return AIMessage(content="ok")
middleware.wrap_model_call(Request(), handler)
- system_content = captured_request["request"].system_message.content
- assert "Middleware recall uses green context" in system_content
- assert "OpenViking active duplicate user turn" not in system_content
- assert "OpenViking active duplicate assistant turn" not in system_content
+ recalled_content = captured_request["request"].messages[0].content
+ assert captured_request["request"].system_message is None
+ assert "Middleware recall uses green context" in recalled_content
+ assert "OpenViking active duplicate user turn" not in recalled_content
+ assert "OpenViking active duplicate assistant turn" not in recalled_content
def test_langgraph_middleware_uses_runtime_thread_id():
@@ -2468,7 +2755,8 @@ def test_langgraph_middleware_uses_runtime_thread_id():
return AIMessage(content="ok")
middleware.wrap_model_call(Request(), handler)
- assert "Runtime thread users prefer teal" in captured_request["request"].system_message.content
+ assert captured_request["request"].system_message is None
+ assert "Runtime thread users prefer teal" in captured_request["request"].messages[0].content
assert client.search_calls[-1]["session_id"] == "runtime-thread"
middleware.after_agent(
diff --git a/integrations/frameworks/langchain/tests/unit/test_langchain_recording.py b/integrations/frameworks/langchain/tests/unit/test_langchain_recording.py
index 0fdb61798..1fa3a306d 100644
--- a/integrations/frameworks/langchain/tests/unit/test_langchain_recording.py
+++ b/integrations/frameworks/langchain/tests/unit/test_langchain_recording.py
@@ -812,7 +812,13 @@ def test_middleware_keeps_context_pending_until_empty_assistant_arrives():
assert any(part["type"] == "context" for part in stored[1]["parts"])
-def test_middleware_retries_failed_commit_without_duplicate_writes():
[email protected]("asynchronous", [False, True])
+async def test_middleware_retries_failed_commit_without_duplicate_writes(asynchronous):
+ from langchain.agents import create_agent
+ from langchain_core.language_models.fake_chat_models import FakeListChatModel
+ from langgraph.checkpoint.memory import InMemorySaver
+
class FailFirstCommitClient(TrackingOpenVikingClient):
def commit_session(self, session_id: str, **kwargs: Any) -> dict[str, Any]:
self.commit_calls.append(session_id)
@@ -821,29 +827,54 @@ def test_middleware_retries_failed_commit_without_duplicate_writes():
return InMemoryOpenVikingClient.commit_session(self, session_id, **kwargs)
client = FailFirstCommitClient()
- middleware = OpenVikingContextMiddleware(
- client=client,
- session_id_resolver=lambda _state, _runtime: "middleware-commit-retry",
- commit_on_after_agent=True,
- )
- state = {
- "messages": [
- HumanMessage(content="Remember this middleware turn."),
- AIMessage(content="Remembered."),
- ]
- }
+ client.records["viking://~/memories/profile.md"] = "Remember azure deployment preferences."
+ checkpointer = InMemorySaver()
+ config = {"configurable": {"thread_id": "middleware-commit-retry"}}
+
+ def build_agent():
+ return create_agent(
+ FakeListChatModel(responses=["Remembered."]),
+ middleware=[
+ OpenVikingContextMiddleware(
+ client=client,
+ target_uri="viking://~/memories",
+ session_id_resolver=lambda _state, _runtime: "middleware-commit-retry",
+ commit_on_after_agent=True,
+ )
+ ],
+ checkpointer=checkpointer,
+ )
+
+ async def run(agent, text):
+ payload = {"messages": [("user", text)]} if text else None
+ return (
+ await agent.ainvoke(payload, config) if asynchronous else agent.invoke(payload, config)
+ )
+ agent = build_agent()
with pytest.raises(OpenVikingPartialWriteError, match="commit failed"):
- middleware.after_agent(state, runtime=None)
- middleware.after_agent(state, runtime=None)
+ await run(agent, "Remember this middleware turn.")
+ await run(agent, None)
- assert client.batch_sizes == [2]
+ assert client.batch_sizes == [1, 1]
assert client.commit_calls == [
"middleware-commit-retry",
"middleware-commit-retry",
]
assert client.sessions["middleware-commit-retry"] == []
assert len(client.archives["middleware-commit-retry"][0]["messages"]) == 2
+ assert not agent.get_state(config).values.get("openviking_pending_context")
+
+ # Persist acknowledged progress even when recovery only retries the commit.
+ await run(build_agent(), "Remember another middleware turn.")
+ archived = [
+ message
+ for archive in client.archives["middleware-commit-retry"]
+ for message in archive["messages"]
+ ]
+ assert client.batch_sizes == [1, 1, 1, 1]
+ assert len(archived) == 4
+ assert sum(part["type"] == "context" for message in archived for part in message["parts"]) == 2
@pytest.mark.parametrize("asynchronous", [False, True])