Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

perf: retrieval, import, LangChain, and session-context optimizations (#3569)

Eurakaxun44c6df262221
diff --git a/openviking/client/local.py b/openviking/client/local.py
index 40bdeb5db..ee6463f0b 100644
--- a/openviking/client/local.py
+++ b/openviking/client/local.py
@@ -767,7 +767,8 @@ class LocalClient(BaseClient):
async def _search():
session = None
- if session_id:
+ # Intent off: skip session.load — SearchService will not scan session either.
+ if session_id and self._service.search.is_intent_enabled():
session = self._service.sessions.session(self._ctx, session_id)
await session.load()
return await self._service.search.search(
@@ -1050,6 +1051,9 @@ class LocalClient(BaseClient):
return {
"session_id": session_id,
"message_count": len(session.messages),
+ # Post-write value so a commit policy can decide without a
+ # follow-up get_session round trip.
+ "pending_tokens": self._session_pending_tokens(session),
}
async def batch_add_messages(
@@ -1113,8 +1117,24 @@ class LocalClient(BaseClient):
"session_id": session_id,
"message_count": len(session.messages),
"added": len(added),
+ # Post-write value so a commit policy can decide without a
+ # follow-up get_session round trip.
+ "pending_tokens": self._session_pending_tokens(session),
}
+ @staticmethod
+ def _session_pending_tokens(session: Any) -> int:
+ """Read the post-write pending-token count from a session.
+
+ Returns 0 when the session object does not expose ``meta`` so callers
+ keep working against lightweight or legacy session implementations.
+ """
+ meta = getattr(session, "meta", None)
+ try:
+ return max(0, int(getattr(meta, "pending_tokens", 0) or 0))
+ except (TypeError, ValueError):
+ return 0
+
def _resolve_message_peer_id(
self,
role: str,
diff --git a/openviking/integrations/langchain/client.py b/openviking/integrations/langchain/client.py
index c6f7dc0c7..ddc3a673f 100644
--- a/openviking/integrations/langchain/client.py
+++ b/openviking/integrations/langchain/client.py
@@ -362,12 +362,28 @@ async def ensure_async_client(
return client
+def is_not_found_error(exc: BaseException) -> bool:
+ """True for OpenViking NOT_FOUND across cli/sdk exception classes."""
+
+ if isinstance(exc, FileNotFoundError):
+ return True
+ return str(getattr(exc, "code", "") or "").upper() == "NOT_FOUND"
+
+
def apply_commit_policy(
client: Any,
session_id: str,
policy: OpenVikingCommitPolicy | None,
+ *,
+ persisted_pending_tokens: Any = None,
) -> dict[str, Any] | None:
- """Apply the configured session commit policy."""
+ """Apply the configured session commit policy after message persistence.
+
+ ``persisted_pending_tokens`` is the post-write value returned by
+ ``add_message`` or ``batch_add_messages``. When an older client does not
+ return it, the pending-token policy falls back to the legacy session lookup.
+ Committing remains a separate operation from message persistence.
+ """
if policy is None or policy.mode == "never":
return None
@@ -380,15 +396,29 @@ def apply_commit_policy(
if policy.mode != "pending_tokens":
raise ValueError(f"Unsupported OpenViking commit policy: {policy.mode}")
- try:
- session = call_openviking(client, "get_session", session_id=session_id, auto_create=False)
- except Exception:
- logger.debug(
- "Skipping OpenViking pending-token commit because session lookup failed",
- exc_info=True,
- )
- return None
- pending_tokens = int(item_value(session, "pending_tokens", 0) or 0)
+ pending_tokens: int | None = None
+ if persisted_pending_tokens is not None:
+ try:
+ pending_tokens = int(persisted_pending_tokens or 0)
+ except (TypeError, ValueError):
+ logger.debug(
+ "Ignoring invalid persisted pending-token value and falling back "
+ "to session lookup: %r",
+ persisted_pending_tokens,
+ )
+
+ if pending_tokens is None:
+ try:
+ session = call_openviking(
+ client, "get_session", session_id=session_id, auto_create=False
+ )
+ except Exception:
+ logger.debug(
+ "Skipping OpenViking pending-token commit because session lookup failed",
+ exc_info=True,
+ )
+ return None
+ pending_tokens = int(item_value(session, "pending_tokens", 0) or 0)
if pending_tokens < policy.pending_token_threshold:
return None
return call_openviking(
@@ -402,6 +432,8 @@ async def aapply_commit_policy(
client: Any,
session_id: str,
policy: OpenVikingCommitPolicy | None,
+ *,
+ persisted_pending_tokens: Any = None,
) -> dict[str, Any] | None:
"""Apply the configured session commit policy without blocking the event loop."""
@@ -416,20 +448,32 @@ async def aapply_commit_policy(
if policy.mode != "pending_tokens":
raise ValueError(f"Unsupported OpenViking commit policy: {policy.mode}")
- try:
- session = await acall_openviking(
- client,
- "get_session",
- session_id=session_id,
- auto_create=False,
- )
- except Exception:
- logger.debug(
- "Skipping OpenViking pending-token commit because session lookup failed",
- exc_info=True,
- )
- return None
- pending_tokens = int(item_value(session, "pending_tokens", 0) or 0)
+ pending_tokens: int | None = None
+ if persisted_pending_tokens is not None:
+ try:
+ pending_tokens = int(persisted_pending_tokens or 0)
+ except (TypeError, ValueError):
+ logger.debug(
+ "Ignoring invalid persisted pending-token value and falling back "
+ "to session lookup: %r",
+ persisted_pending_tokens,
+ )
+
+ if pending_tokens is None:
+ try:
+ session = await acall_openviking(
+ client,
+ "get_session",
+ session_id=session_id,
+ auto_create=False,
+ )
+ except Exception:
+ logger.debug(
+ "Skipping OpenViking pending-token commit because session lookup failed",
+ exc_info=True,
+ )
+ return None
+ pending_tokens = int(item_value(session, "pending_tokens", 0) or 0)
if pending_tokens < policy.pending_token_threshold:
return None
return await acall_openviking(
diff --git a/openviking/integrations/langchain/context.py b/openviking/integrations/langchain/context.py
index 5d57ead20..dd4c4537d 100644
--- a/openviking/integrations/langchain/context.py
+++ b/openviking/integrations/langchain/context.py
@@ -30,6 +30,7 @@ from openviking.integrations.langchain.client import (
ensure_client,
extract_message_text,
get_latest_user_text,
+ is_not_found_error,
)
from openviking.integrations.langchain.history import (
OpenVikingChatMessageHistory,
@@ -201,7 +202,6 @@ class OpenVikingSessionContextAssembler:
query: str = "",
) -> OpenVikingAssembledContext:
client = self._get_client()
- self._ensure_session(client, session_id)
session_context = self._get_session_context(client, session_id)
recall_documents = self._get_recall_documents(
session_id,
@@ -218,7 +218,6 @@ class OpenVikingSessionContextAssembler:
"""Asynchronously assemble session and recall context."""
client = await self._get_async_client()
- await self._aensure_session(client, session_id)
session_context = await self._aget_session_context(client, session_id)
recall_documents = await self._aget_recall_documents(
session_id,
@@ -315,6 +314,9 @@ class OpenVikingSessionContextAssembler:
def _get_session_context(self, client: Any, session_id: str) -> dict[str, Any]:
if not self.include_session_context:
+ # Without a context read there is no NOT_FOUND signal to create from.
+ # Preserve the previous guarantee that recall receives a valid session.
+ self._ensure_session(client, session_id)
return {}
try:
return call_openviking(
@@ -323,7 +325,12 @@ class OpenVikingSessionContextAssembler:
session_id=session_id,
token_budget=self.token_budget,
)
- except Exception:
+ except Exception as exc:
+ if is_not_found_error(exc):
+ # First use: materialize the empty session, but do not issue a second
+ # context read because the newly created session has no context yet.
+ self._ensure_session(client, session_id)
+ return {}
logger.debug("OpenViking session context assembly failed", exc_info=True)
return {}
@@ -333,6 +340,9 @@ class OpenVikingSessionContextAssembler:
session_id: str,
) -> dict[str, Any]:
if not self.include_session_context:
+ # Without a context read there is no NOT_FOUND signal to create from.
+ # Preserve the previous guarantee that recall receives a valid session.
+ await self._aensure_session(client, session_id)
return {}
try:
return await acall_openviking(
@@ -341,7 +351,12 @@ class OpenVikingSessionContextAssembler:
session_id=session_id,
token_budget=self.token_budget,
)
- except Exception:
+ except Exception as exc:
+ if is_not_found_error(exc):
+ # First use: materialize the empty session, but do not issue a second
+ # context read because the newly created session has no context yet.
+ await self._aensure_session(client, session_id)
+ return {}
logger.debug("OpenViking session context assembly failed", exc_info=True)
return {}
diff --git a/openviking/integrations/langchain/history.py b/openviking/integrations/langchain/history.py
index e02820392..ff992e9ca 100644
--- a/openviking/integrations/langchain/history.py
+++ b/openviking/integrations/langchain/history.py
@@ -21,6 +21,7 @@ from openviking.integrations.langchain.client import (
OpenVikingCommitPolicy,
acall_openviking,
call_openviking,
+ is_not_found_error,
)
from openviking.integrations.langchain.messages import (
langchain_message_to_openviking as langchain_message_to_openviking,
@@ -113,9 +114,10 @@ class OpenVikingChatMessageHistory(BaseChatMessageHistory):
session_id=self.session_id,
token_budget=self.token_budget,
)
- except Exception:
+ except Exception as exc:
logger.debug("OpenViking chat history context fetch failed", exc_info=True)
- self._ensure_session(client)
+ if is_not_found_error(exc):
+ self._ensure_session(client)
return []
return restore_openviking_messages(context.get("messages") or [])
@@ -131,9 +133,10 @@ class OpenVikingChatMessageHistory(BaseChatMessageHistory):
session_id=self.session_id,
token_budget=self.token_budget,
)
- except Exception:
+ except Exception as exc:
logger.debug("OpenViking chat history context fetch failed", exc_info=True)
- await self._aensure_session(client)
+ if is_not_found_error(exc):
+ await self._aensure_session(client)
return []
return restore_openviking_messages(context.get("messages") or [])
diff --git a/openviking/integrations/langchain/recording.py b/openviking/integrations/langchain/recording.py
index 1667ff1fb..ca1947b06 100644
--- a/openviking/integrations/langchain/recording.py
+++ b/openviking/integrations/langchain/recording.py
@@ -266,9 +266,10 @@ class OpenVikingSessionRecorder:
messages_written = 0
input_messages_consumed = 0
context_attached = False
+ persisted_pending_tokens: Any = None
for batch in batches:
try:
- call_openviking(
+ write_result = call_openviking(
client,
"batch_add_messages",
session_id=session_id,
@@ -290,13 +291,24 @@ class OpenVikingSessionRecorder:
messages_written += len(batch.payloads)
input_messages_consumed = batch.input_end
context_attached = context_attached or batch.context_attached
+ if isinstance(write_result, dict) and "pending_tokens" in write_result:
+ persisted_pending_tokens = write_result.get("pending_tokens")
+ else:
+ hint = item_value(write_result, "pending_tokens")
+ if hint is not None:
+ persisted_pending_tokens = hint
result = OpenVikingRecordResult(
messages_written=messages_written,
input_messages_consumed=len(input_messages),
context_attached=context_attached,
)
try:
- apply_commit_policy(client, session_id, self.commit_policy)
+ apply_commit_policy(
+ client,
+ session_id,
+ self.commit_policy,
+ persisted_pending_tokens=persisted_pending_tokens,
+ )
except Exception as exc:
self._pending_commit_sessions.add(session_id)
raise OpenVikingPartialWriteError(
@@ -332,9 +344,10 @@ class OpenVikingSessionRecorder:
messages_written = 0
input_messages_consumed = 0
context_attached = False
+ persisted_pending_tokens: Any = None
for batch in batches:
try:
- await acall_openviking(
+ write_result = await acall_openviking(
client,
"batch_add_messages",
session_id=session_id,
@@ -370,13 +383,24 @@ class OpenVikingSessionRecorder:
messages_written += len(batch.payloads)
input_messages_consumed = batch.input_end
context_attached = context_attached or batch.context_attached
+ if isinstance(write_result, dict) and "pending_tokens" in write_result:
+ persisted_pending_tokens = write_result.get("pending_tokens")
+ else:
+ hint = item_value(write_result, "pending_tokens")
+ if hint is not None:
+ persisted_pending_tokens = hint
result = OpenVikingRecordResult(
messages_written=messages_written,
input_messages_consumed=len(input_messages),
context_attached=context_attached,
)
try:
- await aapply_commit_policy(client, session_id, self.commit_policy)
+ await aapply_commit_policy(
+ client,
+ session_id,
+ self.commit_policy,
+ persisted_pending_tokens=persisted_pending_tokens,
+ )
except asyncio.CancelledError as exc:
self._pending_commit_sessions.add(session_id)
_attach_cancellation_progress(
diff --git a/openviking/integrations/langchain/testing.py b/openviking/integrations/langchain/testing.py
index 2f1f2599f..3d93aba12 100644
--- a/openviking/integrations/langchain/testing.py
+++ b/openviking/integrations/langchain/testing.py
@@ -13,6 +13,7 @@ from typing import Any
from openviking.core.peer_id import normalize_peer_id
from openviking.utils.token_estimation import estimate_text_tokens
+from openviking_cli.exceptions import NotFoundError
class InMemoryOpenVikingClient:
@@ -259,6 +260,7 @@ class InMemoryOpenVikingClient:
"session_id": session_id,
"role": role,
"message_count": len(self.sessions[session_id]),
+ "pending_tokens": self.pending_tokens[session_id],
}
def batch_add_messages(
@@ -282,6 +284,7 @@ class InMemoryOpenVikingClient:
"session_id": session_id,
"message_count": len(self.sessions[session_id]),
"added": added,
+ "pending_tokens": self.pending_tokens[session_id],
}
def get_session(self, session_id: str, auto_create: bool = False) -> dict[str, Any]:
@@ -301,6 +304,8 @@ class InMemoryOpenVikingClient:
**_: Any,
) -> dict[str, Any]:
del token_budget
+ if session_id not in self.sessions:
+ raise NotFoundError(session_id, "session")
latest_archive = self.archives.get(session_id, [])[-1:] or []
latest = latest_archive[0] if latest_archive else {}
messages = list(self.sessions.get(session_id, []))
diff --git a/openviking/parse/parsers/excel.py b/openviking/parse/parsers/excel.py
index 79d462d77..c1f23bd6d 100644
--- a/openviking/parse/parsers/excel.py
+++ b/openviking/parse/parsers/excel.py
@@ -8,16 +8,115 @@ Inspired by microsoft/markitdown approach.
"""
import asyncio
+import concurrent.futures
+import time
+from dataclasses import asdict, fields
+from functools import partial
+from multiprocessing import get_context
from pathlib import Path
-from typing import List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
-from openviking.parse.base import ParseResult
+from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result
from openviking.parse.parsers.base_parser import BaseParser
-from openviking_cli.utils.config.parser_config import ParserConfig
+from openviking_cli.utils.config.parser_config import ExcelConfig, ParserConfig
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
+_EXCEL_LAYOUT_EXECUTOR: Optional[concurrent.futures.ProcessPoolExecutor] = None
+_EXCEL_LAYOUT_EXECUTOR_WORKERS: Optional[int] = None
+
+# Non-configurable process-pool internals.
+_EXCEL_PROCESS_POOL_START_METHOD = "spawn"
+_EXCEL_PROCESS_POOL_MIN_BYTES = 200_000
+_EXCEL_PROCESS_POOL_TIMEOUT_S = 120.0
+
+
+def _get_excel_layout_executor(workers: int) -> concurrent.futures.ProcessPoolExecutor:
+ global _EXCEL_LAYOUT_EXECUTOR, _EXCEL_LAYOUT_EXECUTOR_WORKERS
+ workers = max(1, int(workers))
+ if _EXCEL_LAYOUT_EXECUTOR is None:
+ _EXCEL_LAYOUT_EXECUTOR = concurrent.futures.ProcessPoolExecutor(
+ max_workers=workers,
+ mp_context=get_context(_EXCEL_PROCESS_POOL_START_METHOD),
+ )
+ _EXCEL_LAYOUT_EXECUTOR_WORKERS = workers
+ logger.info(
+ f"[ExcelParserProcessPool] Started process pool "
+ f"workers={workers} start_method={_EXCEL_PROCESS_POOL_START_METHOD}"
+ )
+ elif _EXCEL_LAYOUT_EXECUTOR_WORKERS != workers:
+ logger.warning(
+ "[ExcelParserProcessPool] Ignoring process_pool_workers=%s; "
+ "pool already started with workers=%s",
+ workers,
+ _EXCEL_LAYOUT_EXECUTOR_WORKERS,
+ )
+ return _EXCEL_LAYOUT_EXECUTOR
+
+
+def _build_excel_layout_in_process(
+ *,
+ path_str: str,
+ temp_uri: str,
+ instruction: str,
+ layout_kwargs: Dict[str, Any],
+ config_dict: Dict[str, Any],
+ max_rows_per_sheet: int,
+) -> Dict[str, Any]:
+ """CPU-only child process worker.
+
+ The worker must not touch VikingFS, DB, queues, or RequestContext. It only
+ converts Excel to markdown and computes MarkdownParser layout ops.
+ """
+ import asyncio
+
+ from openviking.parse.parsers.excel import ExcelParser
+ from openviking_cli.utils.config.parser_config import ParserConfig
+
+ started = time.perf_counter()
+ path = Path(path_str)
+ allowed_config_fields = {field.name for field in fields(ParserConfig)}
+ filtered_config = {
+ key: value for key, value in config_dict.items() if key in allowed_config_fields
+ }
+ parser = ExcelParser(
+ config=ParserConfig.from_dict(filtered_config),
+ max_rows_per_sheet=max_rows_per_sheet,
+ )
+
+ convert_started = time.perf_counter()
+ if path.suffix.lower() == ".xls":
+ markdown_content = parser._convert_xls_to_markdown(path)
+ else:
+ import openpyxl
+
+ markdown_content = parser._convert_to_markdown(path, openpyxl)
+ convert_s = time.perf_counter() - convert_started
+
+ layout_started = time.perf_counter()
+ layout = asyncio.run(
+ parser._md_parser._compute_layout(
+ markdown_content,
+ temp_uri,
+ source_path=str(path),
+ instruction=instruction,
+ **layout_kwargs,
+ )
+ )
+ layout_s = time.perf_counter() - layout_started
+
+ return {
+ "layout": layout,
+ "convert_s": convert_s,
+ "layout_s": layout_s,
+ "total_s": time.perf_counter() - started,
+ "markdown_chars": len(markdown_content),
+ "layout_ops": len(layout.ops),
+ "layout_write_ops": sum(1 for op in layout.ops if op.kind == "write"),
+ "layout_mkdir_ops": sum(1 for op in layout.ops if op.kind == "mkdir"),
+ }
+
class ExcelParser(BaseParser):
"""
@@ -29,20 +128,28 @@ class ExcelParser(BaseParser):
then delegates to MarkdownParser for tree structure creation.
"""
- def __init__(self, config: Optional[ParserConfig] = None, max_rows_per_sheet: int = 1000):
+ def __init__(
+ self, config: Optional[ParserConfig] = None, max_rows_per_sheet: int = 1000
+ ):
"""
Initialize Excel parser.
Args:
- config: Parser configuration
+ config: Parser configuration (prefer ExcelConfig for process-pool knobs)
max_rows_per_sheet: Maximum rows to process per sheet (0 = unlimited)
"""
from openviking.parse.parsers.markdown import MarkdownParser
self._md_parser = MarkdownParser(config=config)
- self.config = config or ParserConfig()
+ self.config = config or ExcelConfig()
self.max_rows_per_sheet = max_rows_per_sheet
+ def _process_pool_enabled(self) -> bool:
+ return bool(getattr(self.config, "enable_process_pool", False))
+
+ def _process_pool_workers(self) -> int:
+ return max(1, int(getattr(self.config, "process_pool_workers", 2) or 2))
+
@property
def supported_extensions(self) -> List[str]:
return [".xlsx", ".xls", ".xlsm"]
@@ -52,18 +159,7 @@ class ExcelParser(BaseParser):
path = Path(source)
if path.exists():
- # Use xlrd for legacy .xls, openpyxl for .xlsx/.xlsm
- if path.suffix.lower() == ".xls":
- markdown_content = await asyncio.to_thread(self._convert_xls_to_markdown, path)
- else:
- import openpyxl
-
- markdown_content = await asyncio.to_thread(
- self._convert_to_markdown, path, openpyxl
- )
- result = await self._md_parser.parse_content(
- markdown_content, source_path=str(path), instruction=instruction, **kwargs
- )
+ result = await self._parse_existing_path(path, instruction=instruction, **kwargs)
else:
result = await self._md_parser.parse_content(
str(source), instruction=instruction, **kwargs
@@ -72,6 +168,118 @@ class ExcelParser(BaseParser):
result.parser_name = "ExcelParser"
return result
+ async def _parse_existing_path(
+ self, path: Path, instruction: str = "", **kwargs
+ ) -> ParseResult:
+ if self._should_use_process_pool(path, kwargs):
+ try:
+ return await self._parse_existing_path_process_pool(
+ path, instruction=instruction, **kwargs
+ )
+ except Exception as exc:
+ logger.warning(
+ f"[ExcelParserProcessPool] Falling back to in-process parse: {exc}",
+ exc_info=True,
+ )
+
+ # Use xlrd for legacy .xls, openpyxl for .xlsx/.xlsm
+ if path.suffix.lower() == ".xls":
+ markdown_content = await asyncio.to_thread(self._convert_xls_to_markdown, path)
+ else:
+ import openpyxl
+
+ markdown_content = await asyncio.to_thread(self._convert_to_markdown, path, openpyxl)
+ return await self._md_parser.parse_content(
+ markdown_content, source_path=str(path), instruction=instruction, **kwargs
+ )
+
+ def _should_use_process_pool(self, path: Path, kwargs: Dict[str, Any]) -> bool:
+ if not self._process_pool_enabled():
+ return False
+ if path.suffix.lower() == ".xls":
+ return False
+ if kwargs.get("enable_link_rewrite") or kwargs.get("base_dir") or kwargs.get("allowed_media_dirs"):
+ logger.debug("[ExcelParserProcessPool] Disabled for link/media rewrite parse")
+ return False
+ try:
+ return path.stat().st_size >= _EXCEL_PROCESS_POOL_MIN_BYTES
+ except OSError:
+ return False
+
+ async def _parse_existing_path_process_pool(
+ self, path: Path, instruction: str = "", **kwargs
+ ) -> ParseResult:
+ parse_started = time.perf_counter()
+ loop = asyncio.get_running_loop()
+ temp_uri = self._md_parser._create_temp_uri()
+ layout_kwargs = {
+ key: value
+ for key, value in kwargs.items()
+ if key in {"resource_name", "source_name"} and isinstance(value, str)
+ }
+ future = loop.run_in_executor(
+ _get_excel_layout_executor(self._process_pool_workers()),
+ partial(
+ _build_excel_layout_in_process,
+ path_str=str(path),
+ temp_uri=temp_uri,
+ instruction=instruction,
+ layout_kwargs=layout_kwargs,
+ config_dict=asdict(self.config),
+ max_rows_per_sheet=self.max_rows_per_sheet,
+ ),
+ )
+
+ worker_started = time.perf_counter()
+ worker_result = await asyncio.wait_for(
+ future, timeout=_EXCEL_PROCESS_POOL_TIMEOUT_S
+ )
+ worker_s = time.perf_counter() - worker_started
+ layout = worker_result["layout"]
+
+ self._md_parser._rewrite_ctx = {
+ "enabled": False,
+ "source_path": str(path),
+ "doc_name": layout.doc_name,
+ "root_dir": layout.root_dir,
+ "import_root": None,
+ "base_dir": None,
+ "allowed_media_dirs": None,
+ }
+ try:
+ apply_started = time.perf_counter()
+ await self._md_parser._apply_layout(layout)
+ apply_s = time.perf_counter() - apply_started
+ finally:
+ self._md_parser._rewrite_ctx = None
+
+ parse_time = time.perf_counter() - parse_started
+ logger.info(
+ f"[ExcelParserProcessPool] path={path} total={parse_time:.3f}s "
+ f"worker_wall={worker_s:.3f}s convert={worker_result.get('convert_s', -1.0):.3f}s "
+ f"layout={worker_result.get('layout_s', -1.0):.3f}s apply={apply_s:.3f}s "
+ f"chars={worker_result.get('markdown_chars')} ops={worker_result.get('layout_ops')} "
+ f"writes={worker_result.get('layout_write_ops')} mkdirs={worker_result.get('layout_mkdir_ops')}"
+ )
+
+ root = ResourceNode(
+ type=NodeType.ROOT,
+ title=layout.doc_title,
+ level=0,
+ meta=layout.meta.get("frontmatter", {}),
+ )
+ result = create_parse_result(
+ root=root,
+ source_path=str(path),
+ source_format="markdown",
+ parser_name="MarkdownParser",
+ parse_time=parse_time,
+ meta=layout.meta,
+ warnings=layout.warnings,
+ )
+ result.temp_dir_path = layout.temp_uri
+ return result
+
async def parse_content(
self, content: str, source_path: Optional[str] = None, instruction: str = "", **kwargs
) -> ParseResult:
diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.py
index 206cf286b..0fd486f19 100644
--- a/openviking/parse/parsers/markdown.py
+++ b/openviking/parse/parsers/markdown.py
@@ -86,6 +86,13 @@ def _smart_stem(path_or_name: str | Path) -> str:
logger = get_logger(__name__)
+def _env_bool(name: str, default: bool = False) -> bool:
+ raw = os.environ.get(name)
+ if raw is None:
+ return default
+ return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
def _gh_slug(text: str) -> str:
"""GitHub-style heading slug: lowercase, strip punctuation, spaces→'-', keep CJK."""
s = text.strip().lower()
@@ -397,19 +404,80 @@ class MarkdownParser(BaseParser):
"""Phase 2 (write only): replay ``layout.ops`` against the real VikingFS —
create dirs, write each section (rewriting relative links when enabled), then
ingest the local images those sections reference."""
+ started = time.perf_counter()
viking_fs = self._get_viking_fs()
+ mkdir_ops = [op for op in layout.ops if op.kind == "mkdir"]
+ write_ops = [op for op in layout.ops if op.kind == "write"]
+ write_chars = sum(len(op.content or "") for op in write_ops)
+
+ rewrite_enabled = bool(
+ self._rewrite_ctx
+ and self._rewrite_ctx.get("enabled")
+ and self._rewrite_ctx.get("source_path")
+ and self._rewrite_ctx.get("import_root")
+ )
+
+ mkdir_started = time.perf_counter()
+ mkdir_count = 0
for op in layout.ops:
if op.kind == "mkdir":
await viking_fs.mkdir(op.uri, exist_ok=op.exist_ok)
- else:
- await self._write_section(op.uri, op.content)
+ mkdir_count += 1
+ mkdir_s = time.perf_counter() - mkdir_started
+
+ write_started = time.perf_counter()
+ for op in write_ops:
+ await self._write_section(op.uri, op.content)
+ write_s = time.perf_counter() - write_started
# Ingest local image files, placing each image next to the markdown file
- # that references it.
- await self._ingest_local_images(layout.root_dir, base_dir, allowed_media_dirs)
+ # that references it. For generated tables with no image references, avoid
+ # glob+read over every chunk file we just wrote.
+ images_started = time.perf_counter()
+ has_image_refs = self._layout_has_local_image_refs(write_ops)
+ if has_image_refs:
+ await self._ingest_local_images(layout.root_dir, base_dir, allowed_media_dirs)
+ images_s = time.perf_counter() - images_started
+
+ total_s = time.perf_counter() - started
+ if _env_bool("OPENVIKING_MARKDOWN_APPLY_PROFILE", False):
+ logger.info(
+ f"[MarkdownApplyProfile] total={total_s:.3f}s mkdir={mkdir_s:.3f}s "
+ f"write={write_s:.3f}s images={images_s:.3f}s ops={len(layout.ops)} "
+ f"mkdir_ops={len(mkdir_ops)} mkdir_count={mkdir_count} "
+ f"write_ops={len(write_ops)} write_chars={write_chars} "
+ f"rewrite_enabled={rewrite_enabled} has_image_refs={has_image_refs} "
+ f"root={layout.root_dir}"
+ )
# ========== Helper Methods ==========
+ def _layout_has_local_image_refs(self, write_ops: List[_LayoutOp]) -> bool:
+ """Return True when planned markdown content references local images.
+
+ This mirrors the image patterns consumed by _ingest_local_images, but runs
+ against in-memory layout content so pure-text imports avoid a VFS glob/read
+ pass over every generated markdown chunk.
+ """
+ try:
+ from openviking.parse.image_rewrite import HTML_IMG_PATTERN
+ except ImportError:
+ HTML_IMG_PATTERN = re.compile(
+ r"""(<img\s[^>]*?src=["'])([^"']+)(["'][^>]*>)""", re.IGNORECASE
+ )
+
+ for op in write_ops:
+ content = op.content or ""
+ if "![" in content:
+ for match in self._image_pattern.finditer(content):
+ if not self._is_remote_uri(match.group(2)):
+ return True
+ if "<img" in content.lower():
+ for match in HTML_IMG_PATTERN.finditer(content):
+ if not self._is_remote_uri(match.group(2)):
+ return True
+ return False
+
def _extract_frontmatter(self, content: str) -> Tuple[str, Optional[Dict[str, Any]]]:
"""
Extract YAML frontmatter from content.
diff --git a/openviking/parse/registry.py b/openviking/parse/registry.py
index 66931a7bd..dc0140562 100644
--- a/openviking/parse/registry.py
+++ b/openviking/parse/registry.py
@@ -179,7 +179,11 @@ def get_registry() -> ParserRegistry:
"word": config.markdown,
"legacy_doc": config.markdown,
"powerpoint": config.markdown,
- "excel": config.markdown,
+ # Excel had no dedicated config section and reused
+ # ``config.markdown``. Keep unset sectioning fields following
+ # Markdown so existing deployments keep their node structure and
+ # stable URIs after this upgrade.
+ "excel": config.excel.with_sectioning_defaults_from(config.markdown),
"epub": config.markdown,
"image": config.image,
}
diff --git a/openviking/server/mcp_endpoint.py b/openviking/server/mcp_endpoint.py
index 7f8539e83..ac59ea596 100644
--- a/openviking/server/mcp_endpoint.py
+++ b/openviking/server/mcp_endpoint.py
@@ -288,7 +288,8 @@ async def search(
service = get_service()
ctx = _get_ctx()
session = None
- if session_id:
+ # Intent off: skip session.load — SearchService will not scan session either.
+ if session_id and service.search.is_intent_enabled():
session = service.sessions.session(ctx, session_id)
await session.load()
result = await service.search.search(
diff --git a/openviking/server/routers/search.py b/openviking/server/routers/search.py
index d302b8a2b..db37b1109 100644
--- a/openviking/server/routers/search.py
+++ b/openviking/server/routers/search.py
@@ -235,7 +235,8 @@ async def search(
async def _search():
session = None
- if request.session_id:
+ # Intent off: skip session.load — SearchService will not scan session either.
+ if request.session_id and service.search.is_intent_enabled():
session = service.sessions.session(_ctx, request.session_id)
await session.load()
return await service.search.search(
diff --git a/openviking/server/routers/sessions.py b/openviking/server/routers/sessions.py
index 562b9e3a1..e9bed64d7 100644
--- a/openviking/server/routers/sessions.py
+++ b/openviking/server/routers/sessions.py
@@ -138,6 +138,19 @@ def _resolve_message_parts(msg_request: AddMessageRequest) -> List[Part]:
return [TextPart(text=msg_request.content or "")]
+def _session_pending_tokens(session: Any) -> int:
+ """Read the post-write pending-token count from a session.
+
+ Returns 0 when the session object does not expose ``meta`` so the response
+ stays well-formed against lightweight or legacy session implementations.
+ """
+ meta = getattr(session, "meta", None)
+ try:
+ return max(0, int(getattr(meta, "pending_tokens", 0) or 0))
+ except (TypeError, ValueError):
+ return 0
+
+
def _resolve_message_peer_id(msg_request: AddMessageRequest, ctx: RequestContext) -> Optional[str]:
if msg_request.peer_id is not None:
return msg_request.peer_id
@@ -519,6 +532,9 @@ async def add_message(
return {
"session_id": session_id,
"message_count": len(session.messages),
+ # Post-write value so a commit policy can decide without a
+ # follow-up get_session round trip.
+ "pending_tokens": _session_pending_tokens(session),
}
execution = await run_operation(
@@ -567,6 +583,9 @@ async def batch_add_messages(
"session_id": session_id,
"message_count": len(session.messages),
"added": len(msgs),
+ # Post-write value so a commit policy can decide without a
+ # follow-up get_session round trip.
+ "pending_tokens": _session_pending_tokens(session),
}
execution = await run_operation(
diff --git a/openviking/service/search_service.py b/openviking/service/search_service.py
index 58b426ac4..6a6259d09 100644
--- a/openviking/service/search_service.py
+++ b/openviking/service/search_service.py
@@ -48,6 +48,17 @@ class SearchService:
raise NotInitializedError("VikingFS")
return self._viking_fs
+ def is_intent_enabled(self) -> bool:
+ """Whether search uses session context for LLM intent analysis.
+
+ When false, callers should skip session.load / get_context_for_search:
+ VikingFS.search ignores session_info and searches with the raw query.
+ Default is True (matches RetrievalConfig) when config is unset.
+ """
+ if not self._viking_fs or self._viking_fs.retrieval_config is None:
+ return True
+ return bool(self._viking_fs.retrieval_config.enable_intent)
+
async def _resolve_image_url(
self,
image_url: Optional[str],
@@ -98,7 +109,8 @@ class SearchService:
viking_fs = self._ensure_initialized()
session_info = None
- if session and not resolved_image_url:
+ # Intent off: session_info is unused by VikingFS — skip the archive/message scan.
+ if session is not None and self.is_intent_enabled() and not resolved_image_url:
session_info = await session.get_context_for_search(query)
result = await viking_fs.search(
diff --git a/openviking/session/session.py b/openviking/session/session.py
index 9347ef2fd..b5c702048 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -2839,58 +2839,105 @@ class Session:
# ============= Internal methods =============
async def _collect_session_context_components(self) -> Dict[str, Any]:
- """Collect the latest summary archive and merged pending/live messages."""
- archive_states = await self._scan_archive_states()
- completed_archives = [
- state for state in reversed(archive_states) if state.state == "completed"
- ]
- latest_archive = None
- pre_archive_abstracts: List[Dict[str, Any]] = []
- covered_archive_ids = self._covered_archive_ids(archive_states)
- failed_archives = sum(
- state.state == "failed" and state.archive_id not in covered_archive_ids
- for state in archive_states
- )
+ """Collect overview and messages by stopping at the newest terminal archive.
+
+ Archive history grows without bound, so this read path deliberately never
+ walks the whole history. It scans newest → oldest and stops at the first
+ terminal marker (``.done`` or ``.failed.json``):
+
+ - newest terminal is ``completed``: inject that archive's overview when
+ readable, plus raw messages from the newer non-terminal archives;
+ - newest terminal is ``failed``: no overview, and only the newer
+ non-terminal archives contribute raw messages;
+ - no terminal at all: no overview, every archive is still non-terminal
+ so all of their raw messages are returned.
+
+ Nothing at or older than that terminal is read, which is a deliberate
+ deviation from the RFC #3330 recovery formula: an uncovered ``failed``
+ archive no longer replays its raw messages, and only the terminal
+ archive's checkpoints are restored. Phase 2 coverage bookkeeping is
+ unaffected because it keeps using the full ``_scan_archive_states()``
+ scan. Public ``pre_archive_abstracts`` stay empty; abstracts are not read.
+ """
+ archive_refs = await self._list_archive_refs()
+ newer_pending: List[Dict[str, Any]] = []
+ terminal: Optional[Dict[str, Any]] = None
+ terminal_state = ""
+
+ for archive in archive_refs: # newest → oldest
+ state = await self._archive_terminal_state(archive["archive_uri"])
+ if state == "pending":
+ newer_pending.append(archive)
+ continue
+ terminal = archive
+ terminal_state = state
+ break
- for archive in completed_archives:
- # ``.done`` is the authoritative completion marker. A completed
- # archive may intentionally have no overview when Working Memory is
- # disabled, so keep looking for the newest completed archive that
- # actually has a readable overview.
- if latest_archive is None and archive.overview.strip():
+ latest_archive = None
+ failed_archives = 0
+ if terminal is not None and terminal_state == "completed":
+ overview = (await self._read_archive_overview(terminal["archive_uri"])).strip()
+ if overview:
latest_archive = {
- "archive_id": archive.archive_id,
- "archive_uri": archive.archive_uri,
- "overview": archive.overview,
+ "archive_id": terminal["archive_id"],
+ "archive_uri": terminal["archive_uri"],
+ "overview": overview,
"overview_tokens": await self._read_archive_overview_tokens(
- archive.archive_uri, archive.overview
+ terminal["archive_uri"], overview
),
}
- abstract = await self._read_archive_abstract(archive.archive_uri, archive.overview)
- if abstract:
- pre_archive_abstracts.append(
- {
- "archive_id": archive.archive_id,
- "abstract": abstract,
- "tokens": estimate_text_tokens(abstract),
- }
+ else:
+ # A required overview that is missing or unreadable still keeps
+ # the archive terminal here; the warning is emitted by the full
+ # scan used for Phase 2 bookkeeping.
+ logger.warning(
+ "Completed archive has no readable overview: %s",
+ terminal["archive_uri"],
)
+ failed_archives = 1
+ elif terminal is not None:
+ failed_archives = 1
- uncovered = await self._get_uncovered_archive_messages(archive_states)
- merged_messages = self._stable_deduplicate_messages(uncovered + list(self._messages))
- merged_messages = await self._insert_checkpoint_if_needed(
- merged_messages,
- archive_states,
+ archive_messages: List[Message] = []
+ # newer_pending was collected newest-first; restore chronological order.
+ for archive in reversed(newer_pending):
+ try:
+ archive_messages.extend(await self._read_archive_messages(archive["archive_uri"]))
+ except Exception as exc:
+ if not _is_storage_not_found(exc):
+ raise
+ logger.warning(
+ "Skipping pending archive %s because messages.jsonl is missing",
+ archive["archive_uri"],
+ )
+
+ merged_messages = self._stable_deduplicate_messages(
+ archive_messages + list(self._messages)
)
+ merged_messages = await self._insert_terminal_checkpoints(merged_messages, terminal)
return {
"latest_archive": latest_archive,
- "pre_archive_abstracts": pre_archive_abstracts,
- "total_archives": len(archive_states),
+ "pre_archive_abstracts": [],
+ # Directory listing only; deriving an exact count would require the
+ # per-archive marker scan this read path exists to avoid.
+ "total_archives": len(archive_refs),
"failed_archives": failed_archives,
"messages": merged_messages,
}
+ async def _archive_terminal_state(self, archive_uri: str) -> str:
+ """Return ``completed``, ``failed``, or ``pending`` for one archive."""
+ if not self._viking_fs:
+ return "pending"
+ for marker, state in ((".done", "completed"), (".failed.json", "failed")):
+ try:
+ if await self._viking_fs.exists(f"{archive_uri}/{marker}", ctx=self.ctx):
+ return state
+ except Exception:
+ continue
+ return "pending"
+
async def _list_archive_refs(self) -> List[Dict[str, Any]]:
"""List archive refs sorted by archive index descending."""
if not self._viking_fs:
@@ -2946,30 +2993,35 @@ class Session:
)
if done_exists:
- overview = await self._read_archive_overview(archive["archive_uri"])
- if done.get("working_memory_enabled") is True and not overview.strip():
- # New markers distinguish an intentionally overview-less
- # working_memory=false commit from a missing/corrupt
- # required overview. The latter remains logically live and
- # can be rolled forward by a later successful archive.
- logger.warning(
- "Completed archive has no readable required overview: %s",
- archive["archive_uri"],
- )
- states.append(
- ArchiveState(
- archive_id=archive["archive_id"],
- archive_uri=archive["archive_uri"],
- index=archive["index"],
- state="failed",
- done=done,
- failed={
- "stage": "archive_overview",
- "error": "required overview is missing or unreadable",
- },
+ # Only validate overview when Working Memory required one.
+ # Otherwise leave overview empty here and let context assembly
+ # lazy-load the newest terminal completed archive's overview.
+ overview = ""
+ if done.get("working_memory_enabled") is True:
+ overview = await self._read_archive_overview(archive["archive_uri"])
+ if not overview.strip():
+ # New markers distinguish an intentionally overview-less
+ # working_memory=false commit from a missing/corrupt
+ # required overview. The latter remains logically live and
+ # can be rolled forward by a later successful archive.
+ logger.warning(
+ "Completed archive has no readable required overview: %s",
+ archive["archive_uri"],
)
- )
- continue
+ states.append(
+ ArchiveState(
+ archive_id=archive["archive_id"],
+ archive_uri=archive["archive_uri"],
+ index=archive["index"],
+ state="failed",
+ done=done,
+ failed={
+ "stage": "archive_overview",
+ "error": "required overview is missing or unreadable",
+ },
+ )
+ )
+ continue
# working_memory=false legitimately writes .done without an
# overview. Legacy markers lack the explicit flag, so retain
@@ -3362,28 +3414,27 @@ class Session:
)
return records
- async def _insert_checkpoint_if_needed(
+ async def _insert_terminal_checkpoints(
self,
messages: List[Message],
- states: List[ArchiveState],
+ terminal: Optional[Dict[str, Any]],
) -> List[Message]:
- """Insert persisted checkpoint products after retained anchors.
+ """Insert the newest terminal archive's checkpoints after retained anchors.
- Only completed archives are authoritative. Legacy archives without a
+ Only that one archive is read so the cost stays independent of history
+ length. A long User Turn committed partially more than once therefore
+ keeps just its newest compressed prefix; earlier disjoint prefixes from
+ older archives are not restored. Legacy archives without a
``checkpoints`` metadata field do not synthesize one from their overview.
"""
- if not messages:
+ if not messages or terminal is None:
return messages
message_ids = {message.id for message in messages}
candidates: Dict[str, Dict[str, Any]] = {}
- for state in states:
- if state.state != "completed":
- continue
- meta = await self._read_archive_meta(state.archive_uri)
- checkpoints = meta.get("checkpoints")
- if not isinstance(checkpoints, list):
- continue
+ meta = await self._read_archive_meta(terminal["archive_uri"])
+ checkpoints = meta.get("checkpoints")
+ if isinstance(checkpoints, list):
for checkpoint in checkpoints:
if not isinstance(checkpoint, dict):
continue
@@ -3400,14 +3451,11 @@ class Session:
if not valid_source_ids:
continue
- # One long-running User Turn may be partially committed more
- # than once. Keep every disjoint completed prefix instead of
- # letting the newest checkpoint erase earlier compressed Steps.
candidate = candidates.setdefault(
anchor_id,
{
- "archive_id": state.archive_id,
- "archive_uri": state.archive_uri,
+ "archive_id": terminal["archive_id"],
+ "archive_uri": terminal["archive_uri"],
"source_message_ids": [],
"abstracts": [],
},
@@ -3418,8 +3466,6 @@ class Session:
]
if not new_source_ids:
continue
- candidate["archive_id"] = state.archive_id
- candidate["archive_uri"] = state.archive_uri
candidate["source_message_ids"].extend(new_source_ids)
candidate["abstracts"].append(abstract.strip())
@@ -3461,7 +3507,13 @@ class Session:
self,
states: Optional[List[ArchiveState]] = None,
) -> List[Message]:
- """Return pending/failed raw messages not covered by a completed archive."""
+ """Return pending/failed raw messages not covered by a completed archive.
+
+ Kept as the RFC #3330 recovery helper. ``get_session_context`` no longer
+ calls it because that read path stops at the newest terminal archive
+ instead of walking the full history; Phase 2 roll-forward still replays
+ uncovered failed archives through ``_prepare_phase2_archive_messages``.
+ """
states = states if states is not None else await self._scan_archive_states()
covered = self._covered_archive_ids(states)
messages: List[Message] = []
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index bf4bdc669..00c9ea331 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -2249,7 +2249,13 @@ class VikingFS:
except Exception:
target_abstract = ""
- # With session context: intent analysis
+ intent_enabled = (
+ bool(self.retrieval_config.enable_intent)
+ if self.retrieval_config is not None
+ else True
+ )
+
+ # With session context: optional intent analysis
if image_url:
typed_queries = [
TypedQuery(
@@ -2262,7 +2268,7 @@ class VikingFS:
image_query=True,
)
]
- elif session_summary or current_messages:
+ elif intent_enabled and (session_summary or current_messages):
analyzer = IntentAnalyzer(max_recent_messages=5)
with telemetry.measure("search.intent_analysis"):
query_plan = await analyzer.analyze(
@@ -2275,7 +2281,7 @@ class VikingFS:
for tq in typed_queries:
tq.target_directories = retrieval_targets.target_directories
else:
- # No session context: create query directly
+ # No session context, or intent disabled: search with the raw query.
typed_queries = [
TypedQuery(
query=query,
diff --git a/openviking_cli/utils/config/__init__.py b/openviking_cli/utils/config/__init__.py
index 60dd31315..6d879438e 100644
--- a/openviking_cli/utils/config/__init__.py
+++ b/openviking_cli/utils/config/__init__.py
@@ -67,6 +67,7 @@ from .parser_config import (
PARSER_CONFIG_REGISTRY,
AudioConfig,
CodeConfig,
+ ExcelConfig,
HTMLConfig,
ImageConfig,
MarkdownConfig,
@@ -146,6 +147,7 @@ __all__ = [
"AudioConfig",
"VideoConfig",
"MarkdownConfig",
+ "ExcelConfig",
"HTMLConfig",
"TextConfig",
"get_parser_config",
diff --git a/openviking_cli/utils/config/open_viking_config.py b/openviking_cli/utils/config/open_viking_config.py
index dd6eabe6d..65363b9e1 100644
--- a/openviking_cli/utils/config/open_viking_config.py
+++ b/openviking_cli/utils/config/open_viking_config.py
@@ -31,6 +31,7 @@ from .parser_config import (
AudioConfig,
CodeConfig,
DirectoryConfig,
+ ExcelConfig,
FeishuConfig,
HTMLConfig,
ImageConfig,
@@ -205,6 +206,14 @@ class OpenVikingConfig(BaseModel):
default_factory=MarkdownConfig, description="Markdown parsing configuration"
)
+ excel: ExcelConfig = Field(
+ # from_dict on an empty mapping, not the bare constructor: an absent
+ # parsers.excel section must record that no key was set, so sectioning
+ # still follows parsers.markdown for deployments predating this section.
+ default_factory=lambda: ExcelConfig.from_dict({}),
+ description="Excel parsing configuration",
+ )
+
html: HTMLConfig = Field(default_factory=HTMLConfig, description="HTML parsing configuration")
text: TextConfig = Field(default_factory=TextConfig, description="Text parsing configuration")
@@ -379,6 +388,7 @@ class OpenVikingConfig(BaseModel):
"audio",
"video",
"markdown",
+ "excel",
"html",
"text",
"directory",
diff --git a/openviking_cli/utils/config/parser_config.py b/openviking_cli/utils/config/parser_config.py
index 149553896..597144f3d 100644
--- a/openviking_cli/utils/config/parser_config.py
+++ b/openviking_cli/utils/config/parser_config.py
@@ -8,9 +8,9 @@ scattered across different modules. All configurations inherit from ParserConfig
and can be loaded from ov.conf files.
"""
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from pathlib import Path
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, Iterable, Optional, Union
from .config_utils import raise_unknown_config_fields
@@ -452,6 +452,95 @@ class MarkdownConfig(ParserConfig):
raise ValueError("max_heading_depth must be at least 1")
+@dataclass
+class ExcelConfig(ParserConfig):
+ """
+ Configuration for Excel parsing.
+
+ Attributes:
+ enable_process_pool: Offload Excel→Markdown conversion and layout
+ planning to a ProcessPoolExecutor (default off).
+ process_pool_workers: Max worker processes when the pool is enabled.
+ """
+
+ enable_process_pool: bool = False
+ process_pool_workers: int = 2
+
+ # Excel is converted to Markdown and then sectioned by MarkdownParser, so
+ # these fields decide the resulting node structure and stable URIs.
+ _SECTIONING_FIELDS = (
+ "max_content_length",
+ "encoding",
+ "max_section_size",
+ "section_size_flexibility",
+ "max_section_chars",
+ )
+
+ # Names of keys a config source actually provided. Tracked as a plain
+ # instance attribute rather than a dataclass field so it never appears in
+ # asdict/model_dump output, cannot be injected from a config file, and does
+ # not affect equality. Absent means "provenance unknown".
+ _EXPLICIT_ATTR = "_openviking_explicit_keys"
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> "ExcelConfig":
+ """Build the config while remembering which keys were actually present.
+
+ ``with_sectioning_defaults_from`` needs to tell "the user wrote this
+ value" from "the key was absent". Comparing against class defaults
+ cannot do that, so record the provided keys here instead.
+ """
+ config = super().from_dict(data)
+ return config.with_explicit_keys(data)
+
+ def with_explicit_keys(self, names: Iterable[str]) -> "ExcelConfig":
+ """Return this config marked as having ``names`` explicitly configured."""
+ object.__setattr__(self, self._EXPLICIT_ATTR, frozenset(names))
+ return self
+
+ @property
+ def explicit_keys(self) -> Optional[frozenset]:
+ """Keys a config source provided, or ``None`` when unknown."""
+ return getattr(self, self._EXPLICIT_ATTR, None)
+
+ def with_sectioning_defaults_from(self, markdown: "ParserConfig") -> "ExcelConfig":
+ """Inherit sectioning fields that ``parsers.excel`` did not set.
+
+ Excel used to be registered with ``config.markdown`` directly, so a
+ deployment that tuned ``parsers.markdown`` also tuned Excel imports.
+ Introducing a dedicated ``parsers.excel`` section must not silently
+ change that node structure, so a sectioning field absent from
+ ``parsers.excel`` keeps following Markdown. Explicit ``parsers.excel``
+ values always win, including one that happens to equal the class
+ default.
+
+ Configs built without ``from_dict`` carry no key information; those are
+ treated as fully explicit so a hand-constructed ``ExcelConfig`` is never
+ silently rewritten.
+ """
+ if markdown is None:
+ return self
+
+ explicit = self.explicit_keys
+ if explicit is None:
+ return self
+
+ overrides = {
+ name: getattr(markdown, name)
+ for name in self._SECTIONING_FIELDS
+ if hasattr(markdown, name) and name not in explicit
+ }
+ if not overrides:
+ return self
+ return replace(self, **overrides).with_explicit_keys(explicit)
+
+ def validate(self) -> None:
+ """Validate Excel-specific configuration."""
+ super().validate()
+ if self.process_pool_workers < 1:
+ raise ValueError("process_pool_workers must be at least 1")
+
+
@dataclass
class HTMLConfig(ParserConfig):
"""
@@ -678,6 +767,7 @@ PARSER_CONFIG_REGISTRY = {
"audio": AudioConfig,
"video": VideoConfig,
"markdown": MarkdownConfig,
+ "excel": ExcelConfig,
"html": HTMLConfig,
"text": TextConfig,
"directory": DirectoryConfig,
@@ -718,10 +808,10 @@ def get_parser_config(
config_class = PARSER_CONFIG_REGISTRY[parser_type]
- if config_data:
- return config_class.from_dict(config_data)
- else:
- return config_class()
+ # Always go through from_dict, even with no data: configs that track which
+ # keys a source provided need to see the empty mapping to record that none
+ # were set.
+ return config_class.from_dict(config_data or {})
def load_parser_configs_from_dict(config_dict: Dict[str, Any]) -> Dict[str, ParserConfig]:
@@ -751,10 +841,8 @@ def load_parser_configs_from_dict(config_dict: Dict[str, Any]) -> Dict[str, Pars
configs = {}
for parser_type, config_class in PARSER_CONFIG_REGISTRY.items():
- if parser_type in config_dict:
- config_data = config_dict[parser_type]
- configs[parser_type] = config_class.from_dict(config_data)
- else:
- configs[parser_type] = config_class()
+ # from_dict on an empty mapping rather than a bare constructor, so
+ # configs that track provided keys record that a section was absent.
+ configs[parser_type] = config_class.from_dict(config_dict.get(parser_type) or {})
return configs
diff --git a/openviking_cli/utils/config/retrieval_config.py b/openviking_cli/utils/config/retrieval_config.py
index 29179a3c4..b962b6049 100644
--- a/openviking_cli/utils/config/retrieval_config.py
+++ b/openviking_cli/utils/config/retrieval_config.py
@@ -26,5 +26,14 @@ class RetrievalConfig(BaseModel):
"1 uses only the child score."
),
)
+ enable_intent: bool = Field(
+ default=True,
+ description=(
+ "Whether search() loads session context and runs LLM intent analysis / query "
+ "planning when session_id is present. false skips session load, "
+ "get_context_for_search, and IntentAnalyzer — searches with the raw query only "
+ "(same path as no-session search)."
+ ),
+ )
model_config = {"extra": "forbid"}
diff --git a/tests/client/test_rebuild_clients.py b/tests/client/test_rebuild_clients.py
index cc803b6c6..2d545ad1e 100644
--- a/tests/client/test_rebuild_clients.py
+++ b/tests/client/test_rebuild_clients.py
@@ -241,7 +241,14 @@ async def test_local_client_batch_add_messages_forwards_to_session():
],
)
- assert result == {"session_id": "batch-session", "message_count": 2, "added": 2}
+ # A lightweight fake session has no ``meta``, so pending_tokens degrades to 0
+ # instead of raising; the field is always present for commit-policy callers.
+ assert result == {
+ "session_id": "batch-session",
+ "message_count": 2,
+ "added": 2,
+ "pending_tokens": 0,
+ }
assert fake_session.messages[0]["role"] == "user"
assert fake_session.messages[0]["peer_id"] == "explicit-user"
assert fake_session.messages[0]["created_at"] == "2026-05-28T00:00:00+00:00"
@@ -290,7 +297,11 @@ async def test_local_client_add_message_accepts_image_parts():
],
)
- assert result == {"session_id": "image-session", "message_count": 1}
+ assert result == {
+ "session_id": "image-session",
+ "message_count": 1,
+ "pending_tokens": 0,
+ }
assert isinstance(fake_session.messages[0]["parts"][0], TextPart)
assert isinstance(fake_session.messages[0]["parts"][1], ImagePart)
assert fake_session.messages[0]["parts"][1].url == "https://example.com/image.png"
diff --git a/tests/misc/test_retrieval_enable_intent.py b/tests/misc/test_retrieval_enable_intent.py
new file mode 100644
index 000000000..944a38cc6
--- /dev/null
+++ b/tests/misc/test_retrieval_enable_intent.py
@@ -0,0 +1,141 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import contextvars
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from openviking.server.identity import RequestContext, Role
+from openviking.storage.viking_fs import VikingFS
+from openviking_cli.retrieve.types import QueryResult
+from openviking_cli.utils.config.retrieval_config import RetrievalConfig
+from openviking_cli.session.user_id import UserIdentifier
+
+
+def _ctx() -> RequestContext:
+ return RequestContext(user=UserIdentifier("acc1", "user1"), role=Role.USER)
+
+
+def _make_viking_fs(*, enable_intent: bool) -> VikingFS:
+ fs = VikingFS.__new__(VikingFS)
+ fs.agfs = MagicMock()
+ fs.query_embedder = MagicMock(name="embedder")
+ fs.rerank_config = None
+ fs.retrieval_config = RetrievalConfig(enable_intent=enable_intent)
+ fs.vector_store = MagicMock(name="vector_store")
+ fs._bound_ctx = contextvars.ContextVar("vikingfs_bound_ctx_intent_test", default=None)
+ fs._ensure_access = MagicMock()
+ fs._get_vector_store = MagicMock(return_value=fs.vector_store)
+ fs._get_embedder = MagicMock(return_value=fs.query_embedder)
+ fs._ctx_or_default = MagicMock(return_value=_ctx())
+ fs.abstract = AsyncMock(return_value="")
+ return fs
+
+
+def test_retrieval_config_enable_intent_defaults_true():
+ cfg = RetrievalConfig()
+ assert cfg.enable_intent is True
+
+
+def test_retrieval_config_enable_intent_can_disable():
+ cfg = RetrievalConfig(enable_intent=False)
+ assert cfg.enable_intent is False
+
+
+async def test_search_skips_intent_and_uses_raw_query_when_disabled(monkeypatch):
+ fs = _make_viking_fs(enable_intent=False)
+ captured = {}
+
+ class ForbiddenIntentAnalyzer:
+ def __init__(self, *args, **kwargs):
+ raise AssertionError("intent analysis must not run when disabled")
+
+ class FakeRetriever:
+ def __init__(self, storage, embedder, rerank_config, retrieval_config):
+ pass
+
+ async def retrieve(self, typed_query, **kwargs):
+ captured["typed_query"] = typed_query
+ return QueryResult(
+ query=typed_query,
+ matched_contexts=[],
+ searched_directories=typed_query.target_directories,
+ )
+
+ monkeypatch.setattr(
+ "openviking.retrieve.intent_analyzer.IntentAnalyzer",
+ ForbiddenIntentAnalyzer,
+ )
+ monkeypatch.setattr(
+ "openviking.retrieve.hierarchical_retriever.HierarchicalRetriever",
+ FakeRetriever,
+ )
+
+ result = await fs.search(
+ "raw query",
+ target_uri="viking://resources/docs",
+ session_info={
+ "latest_archive_overview": "previous summary",
+ "current_messages": [{"role": "user", "content": "previous turn"}],
+ },
+ ctx=_ctx(),
+ )
+
+ assert result.query_plan is None
+ assert captured["typed_query"].query == "raw query"
+ assert captured["typed_query"].intent == ""
+ assert captured["typed_query"].target_directories == ["viking://resources/docs"]
+
+
+def test_search_service_is_intent_enabled_follows_config():
+ from openviking.service.search_service import SearchService
+
+ svc = SearchService(_make_viking_fs(enable_intent=False))
+ assert svc.is_intent_enabled() is False
+
+ svc.set_viking_fs(_make_viking_fs(enable_intent=True))
+ assert svc.is_intent_enabled() is True
+
+ empty = SearchService()
+ assert empty.is_intent_enabled() is True
+
+
+async def test_search_service_skips_session_context_when_intent_disabled():
+ from openviking.service.search_service import SearchService
+
+ fs = _make_viking_fs(enable_intent=False)
+ fs.search = AsyncMock(
+ return_value=MagicMock(name="find_result", query_plan=None, total=0)
+ )
+ session = MagicMock()
+ session.get_context_for_search = AsyncMock(
+ side_effect=AssertionError("must not scan session when intent disabled")
+ )
+
+ svc = SearchService(fs)
+ await svc.search(query="hello", ctx=_ctx(), session=session, target_uri="")
+
+ session.get_context_for_search.assert_not_awaited()
+ assert fs.search.await_args.kwargs.get("session_info") is None
+
+
+async def test_search_service_loads_session_context_when_intent_enabled():
+ from openviking.service.search_service import SearchService
+
+ fs = _make_viking_fs(enable_intent=True)
+ fs.search = AsyncMock(
+ return_value=MagicMock(name="find_result", query_plan=None, total=0)
+ )
+ session_info = {"latest_archive_overview": "ov", "current_messages": []}
+ session = MagicMock()
+ session.get_context_for_search = AsyncMock(return_value=session_info)
+
+ svc = SearchService(fs)
+ await svc.search(query="hello", ctx=_ctx(), session=session, target_uri="")
+
+ session.get_context_for_search.assert_awaited_once()
+ assert fs.search.await_args.kwargs.get("session_info") is session_info
diff --git a/tests/parse/test_excel_process_pool.py b/tests/parse/test_excel_process_pool.py
new file mode 100644
index 000000000..736d6038e
--- /dev/null
+++ b/tests/parse/test_excel_process_pool.py
@@ -0,0 +1,259 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Tests for ExcelParser._should_use_process_pool, the config-gated routing decision
+that decides whether Excel→Markdown conversion + layout planning run in a
+ProcessPoolExecutor child process instead of the main process's event loop.
+
+A real ProcessPoolExecutor is not exercised here: spawning a child process per
+test is slow and orthogonal to this decision, and the worker itself
+(_build_excel_layout_in_process) is a thin, side-effect-free function that only
+touches its own arguments. This covers the routing decision that gates it.
+"""
+
+from pathlib import Path
+
+import pytest
+
+from openviking.parse.parsers.excel import (
+ ExcelParser,
+ _EXCEL_PROCESS_POOL_MIN_BYTES,
+)
+from openviking_cli.utils.config.parser_config import (
+ ExcelConfig,
+ MarkdownConfig,
+ ParserConfig,
+)
+
+
+class TestShouldUseProcessPool:
+ def _parser(self, **excel_kwargs) -> ExcelParser:
+ return ExcelParser(config=ExcelConfig(**excel_kwargs))
+
+ def _make_file(self, tmp_path: Path, suffix: str = ".xlsx", size: int = 10) -> Path:
+ path = tmp_path / f"sheet{suffix}"
+ path.write_bytes(b"x" * size)
+ return path
+
+ def test_disabled_by_default(self, tmp_path: Path):
+ path = self._make_file(tmp_path, size=_EXCEL_PROCESS_POOL_MIN_BYTES)
+ assert self._parser()._should_use_process_pool(path, {}) is False
+
+ def test_xls_never_uses_process_pool(self, tmp_path: Path):
+ # Legacy .xls goes through xlrd, not the openpyxl path the worker assumes.
+ path = self._make_file(tmp_path, suffix=".xls", size=_EXCEL_PROCESS_POOL_MIN_BYTES)
+ assert (
+ self._parser(enable_process_pool=True)._should_use_process_pool(path, {})
+ is False
+ )
+
+ @pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"enable_link_rewrite": True},
+ {"base_dir": Path(".")},
+ {"allowed_media_dirs": [Path(".")]},
+ ],
+ )
+ def test_link_or_media_rewrite_kwargs_disable_process_pool(self, tmp_path, kwargs):
+ # The worker never touches VikingFS/base_dir-relative media, so any parse
+ # that needs link/media rewriting must stay in-process.
+ path = self._make_file(tmp_path, size=_EXCEL_PROCESS_POOL_MIN_BYTES)
+ assert (
+ self._parser(enable_process_pool=True)._should_use_process_pool(path, kwargs)
+ is False
+ )
+
+ def test_below_min_bytes_returns_false(self, tmp_path):
+ path = self._make_file(tmp_path, size=_EXCEL_PROCESS_POOL_MIN_BYTES - 1)
+ assert (
+ self._parser(enable_process_pool=True)._should_use_process_pool(path, {})
+ is False
+ )
+
+ def test_at_or_above_min_bytes_returns_true(self, tmp_path):
+ path = self._make_file(tmp_path, size=_EXCEL_PROCESS_POOL_MIN_BYTES)
+ assert (
+ self._parser(enable_process_pool=True)._should_use_process_pool(path, {})
+ is True
+ )
+
+
+class TestExcelConfig:
+ def test_defaults(self):
+ cfg = ExcelConfig()
+ assert cfg.enable_process_pool is False
+ assert cfg.process_pool_workers == 2
+
+ def test_validate_rejects_zero_workers(self):
+ cfg = ExcelConfig(process_pool_workers=0)
+ with pytest.raises(ValueError, match="process_pool_workers"):
+ cfg.validate()
+
+ def test_openviking_config_accepts_excel_section(self):
+ from openviking_cli.utils.config.open_viking_config import OpenVikingConfig
+
+ cfg = OpenVikingConfig.from_dict(
+ {
+ "excel": {
+ "enable_process_pool": True,
+ "process_pool_workers": 8,
+ }
+ }
+ )
+ assert cfg.excel.enable_process_pool is True
+ assert cfg.excel.process_pool_workers == 8
+
+
+class TestExcelSectioningInheritance:
+ """Excel used to be registered with ``config.markdown``.
+
+ A dedicated ``parsers.excel`` section must not silently change sectioning,
+ because section boundaries decide node structure and stable Viking URIs.
+ """
+
+ def test_unset_sectioning_follows_markdown(self):
+ markdown = MarkdownConfig(max_section_size=512, max_section_chars=2000)
+ resolved = ExcelConfig.from_dict({}).with_sectioning_defaults_from(markdown)
+ assert resolved.max_section_size == 512
+ assert resolved.max_section_chars == 2000
+
+ def test_explicit_excel_values_win(self):
+ markdown = MarkdownConfig(max_section_size=512, max_section_chars=2000)
+ resolved = ExcelConfig.from_dict(
+ {"max_section_size": 1024}
+ ).with_sectioning_defaults_from(markdown)
+ assert resolved.max_section_size == 1024
+ # Absent from parsers.excel, so this one keeps following Markdown.
+ assert resolved.max_section_chars == 2000
+
+ def test_explicit_value_equal_to_default_is_not_overwritten(self):
+ """"Excel keeps 2048" is a legitimate config even though 2048 is the default.
+
+ Comparing against class defaults cannot distinguish it from an absent
+ key, which would make the documented "explicit values win" contract
+ false for exactly this case.
+ """
+ markdown = MarkdownConfig(max_section_size=512)
+ resolved = ExcelConfig.from_dict(
+ {"max_section_size": 2048}
+ ).with_sectioning_defaults_from(markdown)
+ assert resolved.max_section_size == 2048
+
+ def test_process_pool_knobs_are_never_inherited(self):
+ markdown = MarkdownConfig(max_section_size=512)
+ resolved = ExcelConfig.from_dict(
+ {"enable_process_pool": True, "process_pool_workers": 8}
+ ).with_sectioning_defaults_from(markdown)
+ assert resolved.enable_process_pool is True
+ assert resolved.process_pool_workers == 8
+ assert resolved.max_section_size == 512
+
+ def test_default_markdown_leaves_excel_unchanged(self):
+ resolved = ExcelConfig.from_dict({}).with_sectioning_defaults_from(MarkdownConfig())
+ assert resolved == ExcelConfig()
+
+ def test_missing_markdown_config_is_tolerated(self):
+ assert ExcelConfig.from_dict({}).with_sectioning_defaults_from(None) == ExcelConfig()
+
+ def test_hand_built_config_is_treated_as_fully_explicit(self):
+ """A config built without from_dict carries no key provenance."""
+ markdown = MarkdownConfig(max_section_size=512)
+ resolved = ExcelConfig(max_section_size=2048).with_sectioning_defaults_from(markdown)
+ assert resolved.max_section_size == 2048
+
+ def test_provenance_does_not_affect_equality(self):
+ assert ExcelConfig.from_dict({}) == ExcelConfig()
+ assert ExcelConfig.from_dict({"max_section_size": 2048}) == ExcelConfig()
+
+ def test_absent_excel_section_still_inherits_through_full_config(self):
+ """Deployments predating parsers.excel must keep their Excel sectioning."""
+ from openviking_cli.utils.config.open_viking_config import OpenVikingConfig
+
+ cfg = OpenVikingConfig.from_dict({"markdown": {"max_section_size": 512}})
+ resolved = cfg.excel.with_sectioning_defaults_from(cfg.markdown)
+ assert resolved.max_section_size == 512
+
+ def test_key_provenance_does_not_leak_into_serialization(self):
+ """Provenance is bookkeeping, not config: it must not reach any output.
+
+ A dataclass field would appear in asdict/model_dump and, being a
+ frozenset, would break JSON serialization for callers.
+ """
+ import dataclasses
+ import json
+
+ cfg = ExcelConfig.from_dict({"max_section_size": 2048})
+ dumped = dataclasses.asdict(cfg)
+ assert "_explicit_fields" not in dumped
+ assert not any(key.startswith("_") for key in dumped), sorted(dumped)
+ json.dumps(dumped)
+
+ def test_config_data_cannot_forge_key_provenance(self):
+ """Provenance must not be settable from a config file."""
+ with pytest.raises(ValueError):
+ ExcelConfig.from_dict({"_explicit_fields": ["max_section_size"]})
+
+ def test_replace_and_copy_preserve_key_provenance(self):
+ import copy
+ import dataclasses
+
+ markdown = MarkdownConfig(max_section_size=512)
+ cfg = ExcelConfig.from_dict({"max_section_size": 2048})
+
+ for variant in (
+ dataclasses.replace(cfg, enable_process_pool=True),
+ copy.copy(cfg),
+ copy.deepcopy(cfg),
+ ):
+ resolved = variant.with_sectioning_defaults_from(markdown)
+ assert resolved.max_section_size == 2048
+
+ def test_absent_excel_section_inherits_through_parser_loader(self):
+ """load_parser_configs_from_dict is a second config entry point."""
+ from openviking_cli.utils.config.parser_config import (
+ load_parser_configs_from_dict,
+ )
+
+ configs = load_parser_configs_from_dict({"markdown": {"max_section_size": 512}})
+ resolved = configs["excel"].with_sectioning_defaults_from(configs["markdown"])
+ assert resolved.max_section_size == 512
+
+ def test_explicit_excel_section_wins_through_full_config(self):
+ from openviking_cli.utils.config.open_viking_config import OpenVikingConfig
+
+ cfg = OpenVikingConfig.from_dict(
+ {
+ "markdown": {"max_section_size": 512},
+ "excel": {"max_section_size": 2048, "enable_process_pool": True},
+ }
+ )
+ resolved = cfg.excel.with_sectioning_defaults_from(cfg.markdown)
+ assert resolved.max_section_size == 2048
+ assert resolved.enable_process_pool is True
+
+ def test_registry_resolves_excel_against_markdown(self, monkeypatch):
+ from types import SimpleNamespace
+
+ from openviking.parse import registry as registry_module
+
+ config = SimpleNamespace(
+ text=ParserConfig(),
+ markdown=MarkdownConfig(max_section_size=512, max_section_chars=2000),
+ pdf=ParserConfig(),
+ html=ParserConfig(),
+ excel=ExcelConfig.from_dict({"enable_process_pool": True}),
+ image=ParserConfig(),
+ )
+ monkeypatch.setattr(
+ "openviking_cli.utils.config.get_openviking_config",
+ lambda: config,
+ )
+ monkeypatch.setattr(registry_module, "_default_registry", None)
+
+ excel_parser = registry_module.get_registry().get_parser_for_file("book.xlsx")
+
+ assert excel_parser.config.enable_process_pool is True
+ assert excel_parser.config.max_section_size == 512
+ # The inner MarkdownParser is what actually sections the converted sheet.
+ assert excel_parser._md_parser.config.max_section_size == 512
+ assert excel_parser._md_parser.config.max_section_chars == 2000
diff --git a/tests/parse/test_markdown_apply_layout.py b/tests/parse/test_markdown_apply_layout.py
new file mode 100644
index 000000000..9f1d1279b
--- /dev/null
+++ b/tests/parse/test_markdown_apply_layout.py
@@ -0,0 +1,82 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Tests for MarkdownParser._apply_layout.
+
+``_apply_layout`` replays ``layout.ops`` verbatim: every mkdir op runs, and each
+section is written through ``_write_section`` so it keeps VikingFS's parent-dir
+and encrypted-write handling. These tests pin that contract.
+"""
+
+from unittest.mock import patch
+
+from openviking.parse.parsers.base_parser import BaseParser
+from openviking.parse.parsers.markdown import MarkdownParser, _Layout, _LayoutOp
+
+
+class FakeVikingFS:
+ """Records every mkdir/write call so tests can assert on call counts."""
+
+ def __init__(self):
+ self.mkdir_calls = []
+ self.files = {}
+ self.raw_write_calls = []
+
+ async def mkdir(self, uri, exist_ok=False, **kw):
+ self.mkdir_calls.append(uri)
+
+ async def write(self, uri, data):
+ # Bypasses parent-dir and encrypted-write handling; recorded so a
+ # regression back to this path is visible.
+ self.raw_write_calls.append(uri)
+ self.files[uri] = data
+
+ async def write_file(self, uri, content, **kw):
+ self.files[uri] = content
+
+ async def glob(self, pattern, uri="", **kw):
+ # No images in this layout; let _ingest_local_images short-circuit.
+ return {"matches": []}
+
+
+class TestApplyLayout:
+ def _layout(self) -> _Layout:
+ # One mkdir op is deliberately duplicated to show ops are replayed
+ # verbatim rather than deduplicated.
+ return _Layout(
+ temp_uri="viking://temp/root",
+ root_dir="viking://temp/root/doc",
+ doc_title="doc",
+ doc_name="doc",
+ ops=[
+ _LayoutOp("mkdir", "viking://temp/root"),
+ _LayoutOp("mkdir", "viking://temp/root/doc/sec"),
+ _LayoutOp("mkdir", "viking://temp/root/doc/sec"),
+ _LayoutOp("write", "viking://temp/root/doc/sec/a.md", "A"),
+ _LayoutOp("write", "viking://temp/root/doc/sec/b.md", "B"),
+ _LayoutOp("write", "viking://temp/root/doc/other/c.md", "C"),
+ ],
+ )
+
+ async def test_replays_every_op_and_writes_each_section(self):
+ fake = FakeVikingFS()
+ parser = MarkdownParser()
+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake):
+ await parser._apply_layout(self._layout())
+
+ assert fake.mkdir_calls.count("viking://temp/root/doc/sec") == 2, fake.mkdir_calls
+ assert fake.files == {
+ "viking://temp/root/doc/sec/a.md": "A",
+ "viking://temp/root/doc/sec/b.md": "B",
+ "viking://temp/root/doc/other/c.md": "C",
+ }
+
+ async def test_sections_go_through_write_section_not_raw_write(self):
+ """Section writes must keep VikingFS parent-dir/encrypted-write handling."""
+ fake = FakeVikingFS()
+ parser = MarkdownParser()
+ with patch.object(BaseParser, "_get_viking_fs", return_value=fake):
+ await parser._apply_layout(self._layout())
+
+ assert fake.raw_write_calls == [], (
+ f"Section writes must not bypass write_file: {fake.raw_write_calls}"
+ )
diff --git a/tests/parse/test_markdown_local_image_refs.py b/tests/parse/test_markdown_local_image_refs.py
new file mode 100644
index 000000000..d50375611
--- /dev/null
+++ b/tests/parse/test_markdown_local_image_refs.py
@@ -0,0 +1,43 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Tests for MarkdownParser._layout_has_local_image_refs, the in-memory image-ref
+probe that lets _apply_layout skip _ingest_local_images's glob+read pass over
+every generated markdown chunk when a layout (e.g. an Excel-derived table) has
+no local image references at all."""
+
+from openviking.parse.parsers.markdown import MarkdownParser, _LayoutOp
+
+
+class TestLayoutHasLocalImageRefs:
+ def _parser(self) -> MarkdownParser:
+ return MarkdownParser()
+
+ def _write_ops(self, *contents: str) -> list:
+ return [_LayoutOp("write", f"viking://temp/x/{i}.md", c) for i, c in enumerate(contents)]
+
+ def test_pure_text_returns_false(self):
+ ops = self._write_ops("# Title\n\nJust some plain text with no images at all.")
+ assert self._parser()._layout_has_local_image_refs(ops) is False
+
+ def test_local_markdown_image_returns_true(self):
+ ops = self._write_ops("# Title\n\n![alt](local.png)")
+ assert self._parser()._layout_has_local_image_refs(ops) is True
+
+ def test_remote_markdown_image_returns_false(self):
+ ops = self._write_ops("# Title\n\n![alt](https://example.com/a.png)")
+ assert self._parser()._layout_has_local_image_refs(ops) is False
+
+ def test_local_html_img_returns_true(self):
+ ops = self._write_ops('# Title\n\n<img src="local.png">')
+ assert self._parser()._layout_has_local_image_refs(ops) is True
+
+ def test_mixed_ops_true_if_any_op_has_local_image(self):
+ ops = self._write_ops(
+ "no images here",
+ "https://example.com/remote.png only, still remote: ![alt](https://example.com/remote.png)",
+ "![alt](assets/local.jpg)",
+ )
+ assert self._parser()._layout_has_local_image_refs(ops) is True
+
+ def test_empty_write_ops_returns_false(self):
+ assert self._parser()._layout_has_local_image_refs([]) is False
diff --git a/tests/server/test_api_sessions.py b/tests/server/test_api_sessions.py
index 97ef38585..29f3c39ce 100644
--- a/tests/server/test_api_sessions.py
+++ b/tests/server/test_api_sessions.py
@@ -371,9 +371,17 @@ async def test_get_session_context_includes_incomplete_archive_messages(
]
-async def test_get_session_context_restores_failed_archive_messages(
+async def test_get_session_context_stops_at_newest_failed_archive(
client: httpx.AsyncClient, service
):
+ """The read path stops at the newest terminal archive.
+
+ Archive history grows without bound, so ``get_session_context`` no longer
+ walks it. A newest ``.failed.json`` therefore contributes no overview and no
+ replayed raw messages; the raw file stays durable and Phase 2 roll-forward
+ still absorbs it. This is a deliberate deviation from the RFC #3330
+ ``logical live`` formula.
+ """
create_resp = await client.post("/api/v1/sessions", json={})
session_id = create_resp.json()["result"]["session_id"]
@@ -416,10 +424,13 @@ async def test_get_session_context_restores_failed_archive_messages(
assert resp.status_code == 200
body = resp.json()
assert [m["parts"][0]["text"] for m in body["result"]["messages"]] == [
- "Archived seed",
- "Failed archive message",
"Current live message",
]
+ assert body["result"]["stats"]["failedArchives"] == 1
+
+ # The failed archive's raw messages are still durable on disk.
+ raw = await session._read_archive_messages(failed_archive_uri)
+ assert [message.content for message in raw] == ["Failed archive message"]
async def test_add_message(client: httpx.AsyncClient):
@@ -436,6 +447,44 @@ async def test_add_message(client: httpx.AsyncClient):
assert body["result"]["message_count"] == 1
+async def test_write_responses_return_pending_tokens_for_commit_policy(
+ client: httpx.AsyncClient,
+):
+ """A commit policy must be able to decide from the write response alone.
+
+ Exercises the real REST endpoints rather than a test double, so the
+ ``pending_tokens`` contract is verified where LangChain actually reads it.
+ """
+ create_resp = await client.post("/api/v1/sessions", json={})
+ session_id = create_resp.json()["result"]["session_id"]
+
+ add_resp = await client.post(
+ f"/api/v1/sessions/{session_id}/messages",
+ json=_message_request("user", content="Hello, world!"),
+ )
+ assert add_resp.status_code == 200
+ after_add = add_resp.json()["result"]["pending_tokens"]
+ assert after_add > 0
+
+ batch_resp = await client.post(
+ f"/api/v1/sessions/{session_id}/messages/batch",
+ json={
+ "messages": [
+ _message_request("assistant", content="First reply"),
+ _message_request("user", content="Follow-up question"),
+ ]
+ },
+ )
+ assert batch_resp.status_code == 200
+ after_batch = batch_resp.json()["result"]["pending_tokens"]
+ assert after_batch > after_add
+
+ # The write-returned value must match what get_session would have reported,
+ # which is exactly the extra round trip this field removes.
+ get_resp = await client.get(f"/api/v1/sessions/{session_id}")
+ assert get_resp.json()["result"]["pending_tokens"] == after_batch
+
+
async def test_add_message_accepts_image_part(client: httpx.AsyncClient, service):
create_resp = await client.post("/api/v1/sessions", json={})
session_id = create_resp.json()["result"]["session_id"]
diff --git a/tests/session/test_session_context.py b/tests/session/test_session_context.py
index dbcca26e6..9777af485 100644
--- a/tests/session/test_session_context.py
+++ b/tests/session/test_session_context.py
@@ -445,10 +445,10 @@ class TestGetSessionContext:
assert context["stats"]["activeTokens"] == session.messages[0].estimated_tokens
assert context["stats"]["activeTokens"] > _estimate_tokens("Executing tool...")
- async def test_get_session_context_validates_overviews_and_keeps_public_abstracts_empty(
+ async def test_get_session_context_stops_at_newest_terminal_without_abstracts(
self, client: AsyncOpenViking, monkeypatch
):
- """Completed archives are validated, while the public abstracts field stays empty."""
+ """Newest completed supplies overview; abstracts are never read for context."""
session = client.session(session_id="assemble_lazy_read_test")
summaries = [
"# Summary\n\n" + ("A" * 80),
@@ -471,13 +471,8 @@ class TestGetSessionContext:
session.add_message("user", [TextPart("active tail")])
newest_summary = "# Summary\n\n" + ("C" * 80)
- previous_abstract = "# Summary"
active_tokens = sum(m.estimated_tokens for m in session.messages)
- token_budget = (
- active_tokens
- + _estimate_tokens(newest_summary)
- + (_estimate_tokens(previous_abstract) * 2)
- )
+ token_budget = active_tokens + _estimate_tokens(newest_summary)
original_read_file = session._viking_fs.read_file
read_uris: list[str] = []
@@ -492,23 +487,71 @@ class TestGetSessionContext:
context = await session.get_session_context(token_budget=token_budget)
assert context["latest_archive_overview"] == newest_summary
- # The public field is retained for compatibility but intentionally
- # remains empty; only the latest overview participates in context.
assert context["pre_archive_abstracts"] == []
assert context["stats"]["includedArchives"] == 0
- assert context["stats"]["droppedArchives"] == 3
+ assert context["stats"]["totalArchives"] == 3
- overview_reads = [u for u in read_uris if u.endswith(".overview.md")]
abstract_reads = [u for u in read_uris if u.endswith(".abstract.md")]
- assert all(
- any(f"archive_{index:03d}" in uri for uri in overview_reads)
- for index in (1, 2, 3)
- ), f"Every completed archive overview should be validated, got: {overview_reads}"
- assert all(
- "archive_003" in u or "archive_002" in u or "archive_001" in u for u in abstract_reads
- ), f"Archive abstracts should be read for every returned archive, got: {abstract_reads}"
-
- async def test_get_session_context_drops_oldest_pre_archive_abstracts_first(
+ assert abstract_reads == [], f"Old abstracts must not be read, got: {abstract_reads}"
+ overview_reads = [u for u in read_uris if u.endswith(".overview.md")]
+ assert overview_reads, f"Newest terminal overview should be available, got: {overview_reads}"
+ assert any("archive_003" in u for u in overview_reads), (
+ f"Newest completed overview should be read, got: {overview_reads}"
+ )
+
+ async def test_get_session_context_does_not_touch_archives_older_than_terminal(
+ self, client: AsyncOpenViking, monkeypatch
+ ):
+ """Scan cost must not grow with history length.
+
+ Archive history grows without bound, so the read path stops at the newest
+ terminal marker: no marker, overview, meta or messages read may reference
+ an older archive.
+ """
+ session = client.session(session_id="assemble_terminal_stop_cost_test")
+ summaries = [f"# Summary\n\narchive {index}" for index in range(1, 5)]
+
+ async def fake_generate(self, _messages, latest_archive_overview="", **kwargs):
+ del self, latest_archive_overview, kwargs
+ return summaries.pop(0)
+
+ monkeypatch.setattr(Session, "_generate_archive_summary_async", fake_generate)
+
+ for word in ("first", "second", "third", "fourth"):
+ session.add_message("user", [TextPart(f"{word} turn")])
+ session.add_message("assistant", [TextPart(f"{word} reply")])
+ result = await session.commit_async()
+ await _wait_for_task(result["task_id"])
+
+ session.add_message("user", [TextPart("active tail")])
+
+ touched: list[str] = []
+ original_read_file = session._viking_fs.read_file
+ original_exists = session._viking_fs.exists
+
+ async def tracking_read_file(*args, **kwargs):
+ touched.append(args[0] if args else kwargs.get("uri"))
+ return await original_read_file(*args, **kwargs)
+
+ async def tracking_exists(*args, **kwargs):
+ touched.append(args[0] if args else kwargs.get("uri"))
+ return await original_exists(*args, **kwargs)
+
+ monkeypatch.setattr(session._viking_fs, "read_file", tracking_read_file)
+ monkeypatch.setattr(session._viking_fs, "exists", tracking_exists)
+
+ context = await session.get_session_context()
+
+ assert context["latest_archive_overview"].endswith("archive 4")
+ stale = [
+ uri
+ for uri in touched
+ if isinstance(uri, str)
+ and any(f"archive_{index:03d}" in uri for index in (1, 2, 3))
+ ]
+ assert stale == [], f"Archives older than the terminal must not be read: {stale}"
+
+ async def test_get_session_context_pre_archive_abstracts_always_empty(
self, client: AsyncOpenViking, monkeypatch
):
session = client.session(session_id="assemble_trim_oldest_abstracts_test")
@@ -533,11 +576,8 @@ class TestGetSessionContext:
session.add_message("user", [TextPart("active tail")])
newest_summary = "# Summary\n\n" + ("C" * 80)
- previous_abstract = "# Summary"
active_tokens = sum(m.estimated_tokens for m in session.messages)
- token_budget = (
- active_tokens + _estimate_tokens(newest_summary) + _estimate_tokens(previous_abstract)
- )
+ token_budget = active_tokens + _estimate_tokens(newest_summary)
context = await session.get_session_context(token_budget=token_budget)
@@ -550,7 +590,7 @@ class TestGetSessionContext:
assert context["stats"]["includedArchives"] == 0
assert context["stats"]["droppedArchives"] == 3
- async def test_get_session_context_falls_back_to_older_completed_archive(
+ async def test_get_session_context_newest_done_unreadable_overview_does_not_fallback(
self, client: AsyncOpenViking, monkeypatch
):
session = client.session(session_id="assemble_failed_archive_test")
@@ -585,11 +625,53 @@ class TestGetSessionContext:
context = await session.get_session_context(token_budget=128_000)
- assert context["latest_archive_overview"] == "# Session Summary\n\narchive one"
+ # Newest terminal becomes failed/unusable for overview; no older fallback.
+ assert context["latest_archive_overview"] == ""
assert context["pre_archive_abstracts"] == []
assert context["stats"]["totalArchives"] == 2
assert context["stats"]["includedArchives"] == 0
- assert context["stats"]["droppedArchives"] == 1
+ assert context["stats"]["failedArchives"] == 1
+
+ async def test_get_session_context_newest_failed_skips_overview(
+ self, client: AsyncOpenViking, monkeypatch
+ ):
+ session = client.session(session_id="assemble_newest_failed_test")
+
+ async def fake_generate(self, _messages, latest_archive_overview="", **kwargs):
+ del self, latest_archive_overview, kwargs
+ return "# Session Summary\n\narchive one"
+
+ monkeypatch.setattr(Session, "_generate_archive_summary_async", fake_generate)
+
+ session.add_message("user", [TextPart("turn one")])
+ result = await session.commit_async()
+ await _wait_for_task(result["task_id"])
+
+ await session._viking_fs.write_file(
+ uri=f"{session.uri}/history/archive_002/.failed.json",
+ content='{"stage":"summary","error":"boom"}',
+ ctx=session.ctx,
+ )
+ pending = Message(
+ id="pending-after-fail",
+ role="user",
+ parts=[TextPart("after failed archive")],
+ )
+ await session._viking_fs.write_file(
+ uri=f"{session.uri}/history/archive_003/messages.jsonl",
+ content=pending.to_jsonl() + "\n",
+ ctx=session.ctx,
+ )
+ session.add_message("user", [TextPart("live tail")])
+
+ context = await session.get_session_context(token_budget=128_000)
+
+ assert context["latest_archive_overview"] == ""
+ assert context["pre_archive_abstracts"] == []
+ assert [m["parts"][0]["text"] for m in context["messages"]] == [
+ "after failed archive",
+ "live tail",
+ ]
assert context["stats"]["failedArchives"] == 1
async def test_get_session_context_budget_trim_drops_latest_archive_abstract(
diff --git a/tests/session/test_session_retention_integration.py b/tests/session/test_session_retention_integration.py
index 31148136b..ce374d5d0 100644
--- a/tests/session/test_session_retention_integration.py
+++ b/tests/session/test_session_retention_integration.py
@@ -114,9 +114,14 @@ async def test_session_context_enforces_hard_budget_without_mutating_archive_raw
assert live.content == "B" * 4000
-async def test_failed_archive_returns_raw_but_done_without_overview_stays_archived(
+async def test_context_stops_at_newest_terminal_without_replaying_older_failed_raw(
client: AsyncOpenViking,
):
+ """The read path stops at archive_002 and never replays archive_001 raw.
+
+ Phase 2 still treats the uncovered failed archive as replayable; only
+ ``get_session_context`` stops at the newest terminal.
+ """
session = client.session(session_id="failed_and_wm_disabled_archive_test")
await session.ensure_exists()
await _write_archive(
@@ -134,13 +139,17 @@ async def test_failed_archive_returns_raw_but_done_without_overview_stays_archiv
context = await session.get_session_context()
- assert [message["id"] for message in context["messages"]] == ["u1"]
+ assert [message["id"] for message in context["messages"]] == []
+ # archive_002 is the terminal: .done without a readable overview, so the
+ # read path reports it as failed and injects no overview.
+ assert context["latest_archive_overview"] == ""
assert context["stats"]["failedArchives"] == 1
-async def test_done_with_missing_required_overview_remains_uncovered(
+async def test_done_with_missing_required_overview_reports_failed_without_raw(
client: AsyncOpenViking,
):
+ """A required overview that is unreadable yields no overview and no raw."""
session = client.session(session_id="missing_required_overview_test")
await session.ensure_exists()
await _write_archive(
@@ -156,7 +165,8 @@ async def test_done_with_missing_required_overview_remains_uncovered(
context = await session.get_session_context()
- assert [message["id"] for message in context["messages"]] == ["u1"]
+ assert [message["id"] for message in context["messages"]] == []
+ assert context["latest_archive_overview"] == ""
assert context["stats"]["failedArchives"] == 1
@@ -182,7 +192,8 @@ async def test_legacy_done_marker_covers_only_its_own_archive(
context = await session.get_session_context()
assert context["latest_archive_overview"].endswith("legacy archive two only")
- assert [message["id"] for message in context["messages"]] == ["u1"]
+ # Terminal-stop read path: archive_001 sits at/older than the terminal.
+ assert [message["id"] for message in context["messages"]] == []
async def test_coverage_metadata_cannot_hide_a_pending_archive(
@@ -209,9 +220,15 @@ async def test_coverage_metadata_cannot_hide_a_pending_archive(
},
)
- context = await session.get_session_context()
+ # Phase 2 bookkeeping still refuses to treat a pending archive as covered.
+ states = await session._scan_archive_states()
+ covered = session._covered_archive_ids(states)
+ assert "archive_001" not in covered
- assert [message["id"] for message in context["messages"]] == ["u1"]
+ # The read path stops at the newest terminal, so the pending archive that is
+ # older than it does not contribute raw messages either.
+ context = await session.get_session_context()
+ assert [message["id"] for message in context["messages"]] == []
async def test_later_coverage_absorbs_failed_raw_and_stable_deduplicates_root(
@@ -1051,8 +1068,14 @@ async def test_missing_required_checkpoint_keeps_archive_raw_uncovered(
assert not await session._viking_fs.exists(f"{archive_uri}/.done", ctx=session.ctx)
assert await session._viking_fs.exists(f"{archive_uri}/.failed.json", ctx=session.ctx)
+ # A missing required checkpoint keeps the archive uncovered and durable on
+ # disk. The terminal-stop read path no longer replays its raw messages, so
+ # the archived step ``a1`` is gone while the retained anchor and live tail
+ # stay, and no checkpoint is synthesized.
+ raw = await session._read_archive_messages(archive_uri)
+ assert [message.id for message in raw] == ["u1", "a1"]
context = await session.get_session_context()
- assert [message["id"] for message in context["messages"]] == ["u1", "a1", "a2"]
+ assert [message["id"] for message in context["messages"]] == ["u1", "a2"]
assert not any(message.get("message_kind") == "checkpoint" for message in context["messages"])
@@ -1180,9 +1203,14 @@ async def test_covered_failed_partial_turn_checkpoint_points_to_covering_overvie
assert checkpoint["source_message_ids"] == ["a1"]
-async def test_repeated_partial_commits_merge_checkpoints_for_same_turn_anchor(
+async def test_repeated_partial_commits_restore_only_newest_checkpoint(
client: AsyncOpenViking,
):
+ """Terminal-stop keeps checkpoint reads O(1), so only archive_002 is restored.
+
+ The earlier disjoint prefix from archive_001 is intentionally not merged
+ anymore; restoring it would require walking the whole archive history.
+ """
session = client.session(session_id="repeated_partial_checkpoint_merge_test")
await session.ensure_exists()
anchor = _text_message("u1", "user", "investigate the outage")
@@ -1232,12 +1260,9 @@ async def test_repeated_partial_commits_merge_checkpoints_for_same_turn_anchor(
"a3",
]
checkpoint = context["messages"][1]
- assert checkpoint["source_message_ids"] == ["a1", "a2"]
+ assert checkpoint["source_message_ids"] == ["a2"]
assert checkpoint["parts"][0]["uri"] == second_uri
- assert checkpoint["parts"][0]["abstract"] == (
- "First prefix found pool saturation.\n\n"
- "Second prefix ruled out the network path."
- )
+ assert checkpoint["parts"][0]["abstract"] == "Second prefix ruled out the network path."
async def test_commit_externalizes_tool_outputs_across_the_whole_turn(
@@ -1445,10 +1470,15 @@ async def test_add_waits_for_commit_root_rewrite_and_remains_live(
]
-async def test_queue_enqueue_failure_marks_archive_failed_and_keeps_raw_logically_live(
+async def test_queue_enqueue_failure_marks_archive_failed_and_keeps_raw_durable(
client: AsyncOpenViking,
monkeypatch,
):
+ """The failed marker is authoritative and the raw file stays durable.
+
+ Terminal-stop means ``get_session_context`` no longer replays that raw as
+ logical live; recovery goes through Phase 2 roll-forward instead.
+ """
session = client.session(session_id="queue_enqueue_failure_recovery_test")
session.add_message("user", [TextPart("do not lose me")])
@@ -1474,7 +1504,12 @@ async def test_queue_enqueue_failure_marks_archive_failed_and_keeps_raw_logicall
)
assert states[0].state == "failed"
assert failed["stage"] == "queue_enqueue"
+ # Raw stays durable on disk, and this message is also still live in root
+ # because the failed commit never trimmed it.
+ raw = await session._read_archive_messages(states[0].archive_uri)
+ assert [message.content for message in raw] == ["do not lose me"]
assert [message["parts"][0]["text"] for message in context["messages"]] == ["do not lose me"]
+ assert context["stats"]["failedArchives"] == 1
async def test_phase1_root_rewrite_failure_marks_orphan_archive_failed(
@@ -1507,10 +1542,12 @@ async def test_phase1_root_rewrite_failure_marks_orphan_archive_failed(
context = await fresh.get_session_context()
assert states[0].state == "failed"
assert failed["stage"] == "phase1_persist"
- assert [message["parts"][0]["text"] for message in context["messages"]] == [
- "archive candidate",
- "retained tail",
- ]
+ # The orphan archive keeps its raw messages durable on disk and Phase 2 can
+ # still roll them forward. The terminal-stop read path, however, no longer
+ # replays them as logical live, so only the retained tail is assembled.
+ raw = await fresh._read_archive_messages(states[0].archive_uri)
+ assert [message.content for message in raw] == ["archive candidate"]
+ assert [message["parts"][0]["text"] for message in context["messages"]] == ["retained tail"]
async def test_phase1_enqueues_before_root_rewrite_and_publishes_ready_last(
diff --git a/tests/unit/test_langchain_async_integration.py b/tests/unit/test_langchain_async_integration.py
index 615e6d5d5..2ae49d7a4 100644
--- a/tests/unit/test_langchain_async_integration.py
+++ b/tests/unit/test_langchain_async_integration.py
@@ -1207,6 +1207,62 @@ async def test_async_context_assembler_combines_session_and_recall():
assert assembled.context_parts[0]["uri"] == "viking://resources/runbooks/async.md"
+async def test_async_assemble_skips_create_for_existing_session():
+ """The async path must get the same per-turn call reduction as the sync one."""
+ backing = InMemoryOpenVikingClient()
+ backing.add_message("async-existing", "user", content="Existing async turn.")
+ client = AsyncInMemoryOpenVikingClient(backing)
+ assembler = OpenVikingSessionContextAssembler(
+ async_client=client,
+ target_uri="viking://resources",
+ include_recall=False,
+ )
+
+ await assembler.aassemble(session_id="async-existing")
+ await assembler.aassemble(session_id="async-existing")
+
+ assert client.calls == ["get_session_context", "get_session_context"]
+
+
+async def test_async_assemble_creates_session_only_on_not_found():
+ """A missing session is still materialized, without a second context read."""
+ backing = InMemoryOpenVikingClient()
+ client = AsyncInMemoryOpenVikingClient(backing)
+ assembler = OpenVikingSessionContextAssembler(
+ async_client=client,
+ target_uri="viking://resources",
+ include_recall=False,
+ )
+
+ await assembler.aassemble(session_id="async-missing")
+
+ assert client.calls == ["get_session_context", "create_session"]
+ assert "async-missing" in backing.sessions
+
+
+async def test_async_history_does_not_create_session_on_non_not_found_error():
+ """Only NOT_FOUND may trigger a create, matching the sync error semantics."""
+ backing = InMemoryOpenVikingClient()
+ backing.add_message("async-flaky", "user", content="Existing turn.")
+ client = AsyncInMemoryOpenVikingClient(backing)
+
+ async def failing_get_session_context(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
+ client.calls.append("get_session_context")
+ raise RuntimeError("upstream 503")
+
+ client.get_session_context = failing_get_session_context
+ history = OpenVikingChatMessageHistory(
+ session_id="async-flaky",
+ async_client=client,
+ )
+
+ assert await history.aget_messages() == []
+ assert "create_session" not in client.calls
+
+
@pytest.mark.asyncio
async def test_async_middleware_injects_and_captures_context():
backing = InMemoryOpenVikingClient(
diff --git a/tests/unit/test_langchain_integration.py b/tests/unit/test_langchain_integration.py
index 433af97ab..0e94df152 100644
--- a/tests/unit/test_langchain_integration.py
+++ b/tests/unit/test_langchain_integration.py
@@ -629,7 +629,10 @@ def test_in_memory_openviking_client_batch_add_messages_records_messages():
],
)
- assert result == {"session_id": "batch-session", "message_count": 2, "added": 2}
+ assert result["session_id"] == "batch-session"
+ assert result["message_count"] == 2
+ assert result["added"] == 2
+ assert result["pending_tokens"] > 0
assert [message["role"] for message in client.sessions["batch-session"]] == [
"user",
"assistant",
@@ -773,6 +776,75 @@ def test_session_context_assembler_keeps_peer_id_out_of_recall():
assert "peer_id" not in client.search_calls[-1]
+def test_session_context_assembler_creates_only_after_not_found():
+ class RecordingClient(InMemoryOpenVikingClient):
+ def __init__(self):
+ super().__init__()
+ self.create_calls = 0
+ self.context_calls = 0
+
+ def create_session(self, session_id=None):
+ self.create_calls += 1
+ return super().create_session(session_id=session_id)
+
+ def get_session_context(self, session_id, token_budget=128_000, **kwargs):
+ self.context_calls += 1
+ return super().get_session_context(
+ session_id,
+ token_budget=token_budget,
+ **kwargs,
+ )
+
+ client = RecordingClient()
+ assembler = OpenVikingSessionContextAssembler(
+ client=client,
+ include_recall=False,
+ )
+
+ assembler.assemble(session_id="read-before-create", query="")
+ assembler.assemble(session_id="read-before-create", query="")
+
+ assert client.context_calls == 2
+ assert client.create_calls == 1
+ assert "read-before-create" in client.sessions
+
+
+def test_session_context_assembler_creates_on_code_not_found():
+ class ForeignNotFound(Exception):
+ def __init__(self):
+ super().__init__("session missing")
+ self.code = "NOT_FOUND"
+
+ class RecordingClient(InMemoryOpenVikingClient):
+ def __init__(self):
+ super().__init__()
+ self.create_calls = 0
+
+ def create_session(self, session_id=None):
+ self.create_calls += 1
+ return super().create_session(session_id=session_id)
+
+ def get_session_context(self, session_id, token_budget=128_000, **kwargs):
+ if session_id not in self.sessions:
+ raise ForeignNotFound()
+ return super().get_session_context(
+ session_id,
+ token_budget=token_budget,
+ **kwargs,
+ )
+
+ client = RecordingClient()
+ assembler = OpenVikingSessionContextAssembler(
+ client=client,
+ include_recall=False,
+ )
+
+ assembler.assemble(session_id="foreign-not-found", query="")
+
+ assert client.create_calls == 1
+ assert "foreign-not-found" in client.sessions
+
+
def test_with_openviking_context_wraps_runnable_with_history():
client = InMemoryOpenVikingClient(
{"viking://resources/runbooks/deploy.md": "Azure is the deployment color."}
@@ -1267,6 +1339,30 @@ def test_pending_token_commit_does_not_create_missing_session():
assert "missing-commit-session" not in client.sessions
+def test_pending_token_commit_uses_persisted_value_without_session_lookup():
+ class RecordingClient(InMemoryOpenVikingClient):
+ def __init__(self):
+ super().__init__()
+ self.get_session_calls = 0
+
+ def get_session(self, session_id, auto_create=False):
+ self.get_session_calls += 1
+ return super().get_session(session_id, auto_create=auto_create)
+
+ client = RecordingClient()
+ client.create_session("persisted-hint-session")
+
+ result = apply_commit_policy(
+ client,
+ "persisted-hint-session",
+ OpenVikingCommitPolicy(mode="pending_tokens", pending_token_threshold=1),
+ persisted_pending_tokens=0,
+ )
+
+ assert result is None
+ assert client.get_session_calls == 0
+
+
def test_langgraph_middleware_injects_recall_and_captures_messages():
client = InMemoryOpenVikingClient(
{"viking://user/memories/profile.md": "The user prefers azure deployments."}
@@ -1315,6 +1411,58 @@ def test_langgraph_middleware_injects_recall_and_captures_messages():
assert any(part["type"] == "context" for part in assistant_parts)
+def test_langgraph_middleware_batches_persistence_before_separate_commit():
+ class RecordingClient(InMemoryOpenVikingClient):
+ def __init__(self):
+ super().__init__()
+ self.events = []
+ self._inside_batch = False
+
+ def add_message(self, *args, **kwargs):
+ if not self._inside_batch:
+ self.events.append("add_message")
+ return super().add_message(*args, **kwargs)
+
+ def batch_add_messages(self, *args, **kwargs):
+ self.events.append("batch_add_messages")
+ self._inside_batch = True
+ try:
+ return super().batch_add_messages(*args, **kwargs)
+ finally:
+ self._inside_batch = False
+
+ def get_session(self, *args, **kwargs):
+ self.events.append("get_session")
+ return super().get_session(*args, **kwargs)
+
+ def commit_session(self, *args, **kwargs):
+ self.events.append("commit_session")
+ return super().commit_session(*args, **kwargs)
+
+ client = RecordingClient()
+ middleware = OpenVikingContextMiddleware(
+ client=client,
+ session_id_resolver=lambda state, runtime: "middleware-batch-commit",
+ commit_policy=OpenVikingCommitPolicy(
+ mode="pending_tokens",
+ pending_token_threshold=1,
+ ),
+ )
+
+ middleware.after_agent(
+ {
+ "messages": [
+ HumanMessage(content="Persist me in one batch."),
+ AIMessage(content="Then commit separately."),
+ ]
+ },
+ runtime=None,
+ )
+
+ assert client.events == ["batch_add_messages", "commit_session"]
+ assert client.archives["middleware-batch-commit"]
+
+
def test_langgraph_middleware_uses_peer_id_only_for_message_capture():
client = InMemoryOpenVikingClient(
{"viking://user/memories/profile.md": "The middleware peer prefers azure."}