Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""SemanticProcessor: Processes messages from SemanticQueue, generates .abstract.md and .overview.md."""
import asyncio
import re
import threading
import time
from contextlib import nullcontext
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple
from openviking.core.namespace import classify_uri
from openviking.models.model_error import ModelErrorKind, model_error_kind
from openviking.observability.context import (
bind_root_observability_context,
reset_root_observability_context,
)
from openviking.parse.image_rewrite import (
IMAGE_MAPPINGS_FILENAME,
rewrite_image_uris,
)
from openviking.parse.parsers.constants import (
CODE_EXTENSIONS,
DOCUMENTATION_EXTENSIONS,
FILE_TYPE_CODE,
FILE_TYPE_DOCUMENTATION,
FILE_TYPE_OTHER,
)
from openviking.parse.parsers.media.utils import (
MPEG_TS_PROBE_BYTES,
generate_audio_summary,
generate_image_summary,
generate_video_summary,
get_media_type,
)
from openviking.prompts import render_prompt
from openviking.pyagfs.exceptions import AGFSNotADirectoryError
from openviking.server.identity import RequestContext, Role
from openviking.service.task_tracker import get_task_tracker, run_tracked_operation
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.service.task_work_index import detach_task_context, get_task_context
from openviking.storage.abstract_overview import (
AbstractOverviewWriteResult,
body_for_preview,
freshness_metadata,
markdown_safe_viking_uri,
parse_overview_file_summaries,
plan_abstract_overview_refresh,
write_abstract_overview,
)
from openviking.storage.errors import (
HandoffRecoveryBlocked,
InvalidPathLockRequestError,
LockAcquisitionError,
)
from openviking.storage.index_action import FieldPatch
from openviking.storage.pathlock_recovery import prepare_handoff_retry
from openviking.storage.pathlock_transfer import PathLockGuard
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
from openviking.storage.queuefs.process_result import ProcessResult
from openviking.storage.queuefs.semantic_executor import SemanticTreeExecutor, SemanticTreeStats
from openviking.storage.queuefs.semantic_lock import SemanticLockScope
from openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_coalesce_key
from openviking.storage.queuefs.semantic_ops.freshness_policy import FreshnessAction
from openviking.storage.queuefs.semantic_queue import is_semantic_msg_stale
from openviking.storage.queuefs.semantic_work import SemanticMessageWork, SkillSemanticMessageWork
from openviking.storage.viking_fs import LS_ALL_NODES, SyncDiff, get_viking_fs
from openviking.telemetry import (
bind_telemetry,
bind_telemetry_stage,
)
from openviking.telemetry.span_models import create_root_span_attributes
from openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen
from openviking.utils.ingest_options import IngestOptions
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils import VikingURI
from openviking_cli.utils.config import get_openviking_config
from openviking_cli.utils.logger import get_logger
if TYPE_CHECKING:
from openviking.config.vlm import VLMHandle, VLMResolver
logger = get_logger(__name__)
_PERMANENT_STORAGE_ERRORS = (
FileNotFoundError,
PermissionError,
IsADirectoryError,
NotADirectoryError,
AGFSNotADirectoryError,
InvalidPathLockRequestError,
)
def _is_permanent_storage_error(error: BaseException) -> bool:
return any(isinstance(item, _PERMANENT_STORAGE_ERRORS) for item in (error, error.__cause__))
def _executor_lock(msg: SemanticMsg, lock: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
"""Reuse content-write ownership only when it covers semantic sidecars."""
if msg.generation_trigger != "content_write" or msg.lock_handoff is None:
return lock
covered_paths = msg.lock_handoff.get("covered_paths")
if isinstance(covered_paths, list) and any(
isinstance(item, dict) and item.get("kind") == "tree" for item in covered_paths
):
return lock
return None
class RequestQueueStats:
processed: int = 0
requeue_count: int = 0
error_count: int = 0
class SemanticProcessor(DequeueHandlerBase):
"""
Semantic processor, generates .abstract.md and .overview.md bottom-up.
Processing flow:
1. Concurrently generate summaries for files in directory
2. Collect .abstract.md from subdirectories
3. Generate .abstract.md and .overview.md for this directory
4. Enqueue to EmbeddingQueue for vectorization
"""
_stats_lock = threading.Lock()
@staticmethod
async def _cleanup_local_artifact(msg: SemanticMsg) -> None:
try:
if not msg.artifact_ref:
return
from openviking.parse.output import ParseArtifactRef, store_for_artifact_ref
artifact_ref = ParseArtifactRef.from_dict(msg.artifact_ref)
if artifact_ref.backend != "local":
return
store = store_for_artifact_ref(artifact_ref)
await store.cleanup(artifact_ref)
except Exception as exc:
logger.warning("Failed to clean local parse artifact: %s", exc)
_tree_stats_by_telemetry_id: Dict[str, SemanticTreeStats] = {}
_tree_stats_by_uri: Dict[str, SemanticTreeStats] = {}
_tree_stats_order: List[Tuple[str, str]] = []
_request_stats_by_telemetry_id: Dict[str, RequestQueueStats] = {}
_request_stats_order: List[str] = []
_max_cached_stats = 256
def __init__(
self,
max_concurrent_llm: int = 32,
*,
embedding_worker_stopped: Optional[Callable[[], bool]] = None,
vlm_resolver: Optional["VLMResolver"] = None,
):
"""
Initialize SemanticProcessor.
Args:
max_concurrent_llm: Maximum concurrent LLM calls
"""
self.max_concurrent_llm = max_concurrent_llm
self._default_ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
self._circuit_breakers: Dict[str, CircuitBreaker] = {}
self._breaker_lock = threading.Lock()
self._embedding_worker_stopped = embedding_worker_stopped
self._vlm_resolver = vlm_resolver
def _account_breaker(self, account_id: str) -> CircuitBreaker:
with self._breaker_lock:
breaker = self._circuit_breakers.get(account_id)
if breaker is None:
breaker = self._circuit_breakers[account_id] = CircuitBreaker()
return breaker
async def _get_vlm_config(self, ctx: RequestContext) -> "VLMHandle":
if self._vlm_resolver is None:
raise RuntimeError("SemanticProcessor requires a VLM resolver for account-owned work")
return await self._vlm_resolver.get_vlm(ctx.account_id)
@classmethod
def _cache_tree_stats(cls, telemetry_id: str, uri: str, stats: SemanticTreeStats) -> None:
with cls._stats_lock:
if telemetry_id:
previous = cls._tree_stats_by_telemetry_id.get(telemetry_id)
if previous is not None:
stats = SemanticTreeStats(
total_nodes=previous.total_nodes + stats.total_nodes,
pending_nodes=previous.pending_nodes + stats.pending_nodes,
in_progress_nodes=previous.in_progress_nodes + stats.in_progress_nodes,
done_nodes=previous.done_nodes + stats.done_nodes,
failures=[*previous.failures, *stats.failures],
indexed_records=previous.indexed_records + stats.indexed_records,
)
cls._tree_stats_by_telemetry_id[telemetry_id] = stats
cls._tree_stats_by_uri[uri] = stats
cls._tree_stats_order.append((telemetry_id, uri))
if len(cls._tree_stats_order) > cls._max_cached_stats:
old_telemetry_id, old_uri = cls._tree_stats_order.pop(0)
if old_telemetry_id:
cls._tree_stats_by_telemetry_id.pop(old_telemetry_id, None)
cls._tree_stats_by_uri.pop(old_uri, None)
@classmethod
def consume_tree_stats(
cls,
telemetry_id: str = "",
uri: Optional[str] = None,
) -> Optional[SemanticTreeStats]:
with cls._stats_lock:
if telemetry_id and telemetry_id in cls._tree_stats_by_telemetry_id:
stats = cls._tree_stats_by_telemetry_id.pop(telemetry_id, None)
if uri:
cls._tree_stats_by_uri.pop(uri, None)
return stats
if uri and uri in cls._tree_stats_by_uri:
return cls._tree_stats_by_uri.pop(uri, None)
return None
@classmethod
def _merge_request_stats(
cls,
telemetry_id: str,
processed: int = 0,
requeue_count: int = 0,
error_count: int = 0,
) -> None:
if not telemetry_id:
return
with cls._stats_lock:
stats = cls._request_stats_by_telemetry_id.setdefault(telemetry_id, RequestQueueStats())
stats.processed += processed
stats.requeue_count += requeue_count
stats.error_count += error_count
cls._request_stats_order.append(telemetry_id)
if len(cls._request_stats_order) > cls._max_cached_stats:
old_telemetry_id = cls._request_stats_order.pop(0)
if old_telemetry_id != telemetry_id:
cls._request_stats_by_telemetry_id.pop(old_telemetry_id, None)
@classmethod
def consume_request_stats(cls, telemetry_id: str) -> Optional[RequestQueueStats]:
if not telemetry_id:
return None
with cls._stats_lock:
return cls._request_stats_by_telemetry_id.pop(telemetry_id, None)
@staticmethod
def _ctx_from_semantic_msg(msg: SemanticMsg) -> RequestContext:
return RequestContext(
user=UserIdentifier(msg.account_id, msg.user_id),
role=Role(msg.role),
group_ids=tuple(msg.group_ids),
bypass_acl=True,
)
def _detect_file_type(self, file_name: str) -> str:
"""
Detect file type for summary prompt selection.
Args:
file_name: File name with extension
Returns:
FILE_TYPE_CODE, FILE_TYPE_DOCUMENTATION, or FILE_TYPE_OTHER
"""
file_name_lower = file_name.lower()
# Documentation prompts should win over broad code skeleton recognition.
for ext in DOCUMENTATION_EXTENSIONS:
if file_name_lower.endswith(ext):
return FILE_TYPE_DOCUMENTATION
from openviking.parse.parsers.code.ast.providers import supports_code_skeleton
if supports_code_skeleton(file_name):
return FILE_TYPE_CODE
# Keep legacy extension-based routing for code-like text formats not
# covered by tags queries or tree-sitter-language-pack.
for ext in CODE_EXTENSIONS:
if file_name_lower.endswith(ext):
return FILE_TYPE_CODE
# Default to other
return FILE_TYPE_OTHER
async def _reenqueue_semantic_msg(
self,
msg: SemanticMsg,
*,
enqueue: Optional[Callable[[Any, SemanticMsg], Awaitable[None]]] = None,
) -> None:
"""Re-enqueue a semantic message for later processing.
Throttles with a sleep when the circuit breaker is open to prevent
re-enqueue storms (messages cycling at 5/sec during OPEN window).
"""
import asyncio
from openviking.storage.queuefs import get_queue_manager
# Throttle to prevent re-enqueue storm during OPEN window
wait = self._account_breaker(msg.account_id).retry_after
if wait > 0:
await asyncio.sleep(wait)
queue_manager = get_queue_manager()
if queue_manager is not None:
semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC)
if enqueue is None:
await semantic_queue.enqueue(msg)
else:
await enqueue(semantic_queue, msg)
logger.info(f"Re-enqueued semantic message: {msg.uri}")
else:
logger.warning(f"No queue manager available, cannot re-enqueue: {msg.uri}")
async def _enqueue_semantic_retry(
self, queue, msg: SemanticMsg, scope: SemanticLockScope
) -> None:
"""Transfer the live semantic lease to a retry before releasing this worker.
Reusing the consumed handoff would require acquiring an unrelated lock,
which conflicts with an update request still waiting under its outer lease.
"""
agfs = get_viking_fs()._async_agfs
lock_guard = PathLockGuard(agfs, scope.lock)
handoff = await lock_guard.to_handoff()
msg.lock_handoff = handoff
def accept(result: Any) -> bool:
if result == "deduplicated":
return False
scope.lock = None
scope._owned = False
return True
try:
enqueue_id = await lock_guard.handoff_and_dispatch(
handoff,
lambda: queue.enqueue(msg, deduplicate_memory=False),
accepted=accept,
)
if enqueue_id == "deduplicated":
raise RuntimeError("Semantic retry was unexpectedly deduplicated")
finally:
if lock_guard.lease is not None:
scope.lock = lock_guard.lease
scope._owned = True
async def _enqueue_skill_retry(self, queue, msg: SemanticMsg, scope: SemanticLockScope) -> None:
"""Compatibility wrapper for existing skill retry callers."""
await self._enqueue_semantic_retry(queue, msg, scope)
async def _requeue_semantic_msg_after_error(
self,
msg: SemanticMsg,
error: Exception,
*,
work: SemanticMessageWork,
) -> ProcessResult:
if isinstance(error, HandoffRecoveryBlocked):
payload = await prepare_handoff_retry(msg, error, queue="Semantic")
if payload is None:
self._merge_request_stats(msg.telemetry_id, error_count=1)
await self._cleanup_local_artifact(msg)
return ProcessResult.failed(f"Pathlock recovery retry limit reached: {error}")
msg.lock_handoff_retry = payload["lock_handoff_retry"]
try:
await work.reenqueue_after_error()
self._merge_request_stats(msg.telemetry_id, requeue_count=1)
except Exception as requeue_err:
if isinstance(error, HandoffRecoveryBlocked):
# No replacement is durable: propagate so the original delivery
# remains recoverable rather than being ACKed as failed.
raise
logger.error(f"Failed to re-enqueue semantic message: {requeue_err}")
self._merge_request_stats(msg.telemetry_id, error_count=1)
await self._cleanup_local_artifact(msg)
return ProcessResult.failed(str(error))
return ProcessResult.requeued()
async def _enqueue_parent_refresh(
self, msg: SemanticMsg, uri: str, *, l0_body_changed: bool
) -> None:
if msg.generation_trigger == "content_copy":
return
if msg.context_type not in {"resource", "skill"}:
return
if not msg.propagate_to_parent:
return
parent = VikingURI(uri).parent
if parent is None:
return
parent_uri = parent.uri.rstrip("/")
if msg.context_type == "skill":
classification = classify_uri(parent_uri)
if (
not classification.is_skill
or classification.is_skill_namespace
or classification.is_skill_root
):
return
if (
not parent_uri
or parent_uri in {"viking://", "viking:", "viking://user", "viking://agent"}
or parent_uri == uri.rstrip("/")
):
return
parent_ctx = self._ctx_from_semantic_msg(msg)
semantic_config = get_openviking_config().semantic
try:
decision = await plan_abstract_overview_refresh(
viking_fs=get_viking_fs(),
dir_uri=parent_uri,
changed_entries=1,
ctx=parent_ctx,
l0_body_changed=l0_body_changed,
# This helper handles automatic upward propagation only. Manual
# refresh/ingest bypasses the threshold for its requested root,
# not for every ancestor reached afterwards.
force_refresh=False,
overview_sample_limit=getattr(semantic_config, "overview_sample_limit", 32),
refresh_ratio=getattr(semantic_config, "freshness_refresh_ratio", 0.10),
lock_timeout_secs=1.0,
)
except LockAcquisitionError:
logger.info(
"Skipping best-effort parent freshness update because sidecars are busy: %s",
parent_uri,
)
return
if decision.action is not FreshnessAction.REFRESH_NOW:
logger.debug(
"Parent semantic refresh %s for %s (pending=%d, total=%d)",
decision.action.value,
parent_uri,
decision.pending_after,
decision.total_entries,
)
return
from openviking.storage.queuefs import get_queue_manager
queue_manager = get_queue_manager()
semantic_queue = queue_manager.get_queue(queue_manager.SEMANTIC, allow_create=True)
parent_msg = SemanticMsg(
uri=parent_uri,
context_type=msg.context_type,
recursive=False,
account_id=msg.account_id,
user_id=msg.user_id,
group_ids=msg.group_ids,
peer_id=msg.peer_id,
role=msg.role,
skip_vectorization=msg.skip_vectorization,
changes={"modified": [uri]},
generation_trigger="parent_refresh",
coalesce_key=build_semantic_coalesce_key(
context_type=msg.context_type,
uri=parent_uri,
account_id=msg.account_id,
user_id=msg.user_id,
peer_id=msg.peer_id,
),
)
with detach_task_context():
await semantic_queue.enqueue(parent_msg)
logger.info("Enqueued parent semantic refresh: %s", parent_uri)
def _message_work(
self, msg: SemanticMsg, lock: Optional[Dict[str, Any]] = None
) -> SemanticMessageWork:
work_type = SkillSemanticMessageWork if msg.context_type == "skill" else SemanticMessageWork
return work_type(self, msg, lock)
async def on_dequeue(
self,
data: Optional[Dict[str, Any]],
lock: Optional[Dict[str, Any]] = None,
) -> ProcessResult:
"""Process dequeued SemanticMsg, recursively process all subdirectories."""
import json
if not data:
return ProcessResult.success()
try:
if "data" in data and isinstance(data["data"], str):
data = json.loads(data["data"])
msg = SemanticMsg.from_dict(data)
except Exception as exc:
logger.error("Invalid semantic message: %s", exc, exc_info=True)
return ProcessResult.failed(str(exc))
if msg.context_type != "skill" or get_task_context() is not None:
return await self._process_message(msg, lock)
# Parent refreshes and recovered taskless Skill messages need their own
# child-work collector so package ownership lasts through vector writes.
result: Optional[ProcessResult] = None
async def process() -> Dict[str, Any]:
nonlocal result
result = await self._process_message(msg, lock)
if result.error is not None:
context = get_task_context()
assert context is not None
await get_task_tracker().fail(
context.task_id,
result.error,
account_id=msg.account_id,
user_id=msg.user_id,
)
return {"uri": msg.uri, "outcome": result.outcome.value}
await run_tracked_operation(
"semantic_refresh",
account_id=msg.account_id,
user_id=msg.user_id,
resource_id=msg.uri,
operation=process,
wait=False,
)
assert result is not None
return result
async def _process_message(
self,
msg: SemanticMsg,
lock: Optional[Dict[str, Any]],
) -> ProcessResult:
collector = None
work: Optional[SemanticMessageWork] = None
execute_started_at: float | None = None
queue_wait_ms = 0.0
execute_status = "ok"
breaker = None
try:
breaker = self._account_breaker(msg.account_id)
work = self._message_work(msg, lock)
work.start()
execute_started_at = time.perf_counter()
if msg.queue_enqueued_at > 0:
queue_wait_ms = max((time.time() - msg.queue_enqueued_at) * 1000.0, 0.0)
if VikingURI(msg.uri).parent is None:
logger.warning("Skipping semantic generation for root URI: %s", msg.uri)
await self._cleanup_local_artifact(msg)
return ProcessResult.success()
if is_semantic_msg_stale(msg):
live_file_changes = {
kind: list(msg.changes.get(kind, []))
for kind in ("added", "modified")
if msg.changes and msg.changes.get(kind)
}
if msg.generation_trigger == "content_write" and live_file_changes:
# Coalescing replaces directory aggregation, not per-file
# maintenance. Let the newest message aggregate while this
# one still summarizes/vectorizes its changed files.
logger.info(
"Downgrading stale semantic message to file-only work: uri=%s version=%s",
msg.uri,
msg.coalesce_version,
)
msg.aggregate_directory = False
msg.changes = live_file_changes
msg.coalesce_key = ""
msg.coalesce_version = 0
else:
logger.info(
"Skipping stale semantic message: uri=%s version=%s",
msg.uri,
msg.coalesce_version,
)
await work.skip()
await self._cleanup_local_artifact(msg)
return ProcessResult.success()
# Circuit breaker: if API is known-broken, re-enqueue and wait
try:
breaker.check()
except CircuitBreakerOpen:
logger.warning(
f"Circuit breaker is open, re-enqueueing semantic message: {msg.uri}"
)
await work.reenqueue()
self._merge_request_stats(msg.telemetry_id, requeue_count=1)
return ProcessResult.requeued()
collector = work.resolve_telemetry()
telemetry_ctx = bind_telemetry(collector) if collector is not None else nullcontext()
with telemetry_ctx:
root_attrs = create_root_span_attributes(
http_method="QUEUE",
http_route=msg.context_type or "/queuefs/semantic",
request_id=msg.telemetry_id or msg.id,
url_path=msg.uri,
)
root_attrs.account_id = msg.account_id
root_attrs.user_id = msg.user_id
root_context_token = bind_root_observability_context(root_attrs)
try:
current_ctx = self._ctx_from_semantic_msg(msg)
logger.info(
f"Processing semantic generation for: {msg.uri} (recursive={msg.recursive})"
)
logger.debug("Processing semantic message id=%s uri=%s", msg.id, msg.uri)
# Resolving a deleted root can recreate it for lock metadata.
# Settle queued ownership before acknowledging skipped work.
if not await get_viking_fs().exists(msg.uri, ctx=current_ctx):
logger.info("Skipping semantic message for missing root: uri=%s", msg.uri)
await work.skip()
return ProcessResult.success()
if self._vlm_resolver is None:
raise RuntimeError(
"SemanticProcessor requires a VLM resolver for account-owned work"
)
if not await work.acquire_lock(current_ctx):
return ProcessResult.success()
semantic_lock = work.scope
assert semantic_lock is not None
dag_stats = None
processing_succeeded = False
try:
if msg.plan is not None:
if msg.uri.rstrip("/") != msg.plan.root_uri:
raise ValueError("semantic message URI must match plan root_uri")
if msg.context_type != msg.plan.context_type:
raise ValueError(
"semantic message context_type must match semantic plan"
)
for run_uri in msg.plan.execution_root_uris():
executor = SemanticTreeExecutor(
processor=self,
context_type=msg.context_type,
max_concurrent_llm=self.max_concurrent_llm,
ctx=current_ctx,
lock=semantic_lock.lock,
source=msg.plan.source_metadata,
semantic_plan=msg.plan,
telemetry_id=msg.telemetry_id,
generation_trigger=msg.generation_trigger,
)
await executor.run(run_uri)
self._cache_tree_stats(
msg.telemetry_id, run_uri, executor.get_stats()
)
if not executor.stale:
write_result = getattr(
executor,
"root_write_result",
AbstractOverviewWriteResult(
wrote=True, abstract_body_changed=True
),
)
await self._enqueue_parent_refresh(
msg,
run_uri,
l0_body_changed=write_result.abstract_body_changed,
)
from collections import Counter
entries = msg.plan.tree.entries
action_counts = Counter(
entry.semantic_action.value for entry in entries
)
vector_slots = sum(
slot.action.value in {"upsert", "merge"}
for entry in entries
for slot in entry.index_slots
)
logger.debug(
"[SemanticPlanExecution] root=%s execution_roots=%d "
"semantic_entries=%d actions=%s planned_vector_upserts=%d",
msg.plan.root_uri,
len(msg.plan.execution_root_uris()),
len(entries),
dict(action_counts),
vector_slots,
)
# Regular memory writes keep their specialized update path.
# Callers must explicitly opt into directory aggregation; the
# trigger remains descriptive metadata, not an algorithm switch.
elif msg.context_type == "memory" and not msg.use_hierarchical_aggregation:
await self._process_memory_directory(
msg,
ctx=current_ctx,
lock=semantic_lock.lock,
)
else:
is_incremental = False
target_uri = msg.target_uri
run_uri = msg.uri
changes = msg.changes
viking_fs = get_viking_fs()
if msg.target_uri:
target_exists = await viking_fs.exists(
msg.target_uri, ctx=current_ctx
)
if msg.uri != msg.target_uri:
logger.info(
"Syncing semantic source into target before processing: "
f"{msg.uri} -> {msg.target_uri}"
)
diff = await work.run_write(
lambda: self._sync_topdown_recursive(
msg.uri,
msg.target_uri,
ctx=current_ctx,
lock=semantic_lock.lock,
)
)
logger.info(
"[SyncDiff] Diff computed: "
f"added_files={len(diff.added_files)}, "
f"deleted_files={len(diff.deleted_files)}, "
f"updated_files={len(diff.updated_files)}, "
f"added_dirs={len(diff.added_dirs)}, "
f"deleted_dirs={len(diff.deleted_dirs)}"
)
changes = diff.to_changes()
is_incremental = True
target_uri = msg.target_uri
run_uri = msg.target_uri
elif (
target_exists
and msg.changes is not None
and msg.uri == msg.target_uri
):
is_incremental = True
logger.info(
f"Using direct incremental semantic update for: {msg.uri}"
)
elif msg.changes is not None:
is_incremental = True
target_uri = msg.uri
logger.info(
f"Using direct incremental semantic update for: {msg.uri}"
)
executor = SemanticTreeExecutor(
processor=self,
context_type=msg.context_type,
max_concurrent_llm=self.max_concurrent_llm,
ctx=current_ctx,
incremental_update=is_incremental,
target_uri=target_uri,
recursive=msg.recursive,
lock=_executor_lock(msg, semantic_lock.lock),
is_code_repo=msg.is_code_repo,
changes=changes,
skip_vectorization=msg.skip_vectorization,
ingest_options=msg.ingest_options,
coalesce_key=msg.coalesce_key,
coalesce_version=msg.coalesce_version,
source=msg.source,
generation_trigger=msg.generation_trigger,
aggregate_directory=msg.aggregate_directory,
copy_source_uri=msg.copy_source_uri,
file_md5s=msg.file_md5s,
artifact_files=msg.artifact_files,
file_abstracts=msg.file_abstracts,
file_vector_actions=msg.file_vector_actions,
file_ingest_options=msg.file_ingest_options,
telemetry_id=msg.telemetry_id,
)
await executor.run(run_uri)
dag_stats = executor.get_stats()
self._cache_tree_stats(
msg.telemetry_id,
run_uri,
dag_stats,
)
if not executor.stale and msg.aggregate_directory:
write_result = getattr(
executor,
"root_write_result",
AbstractOverviewWriteResult(
wrote=True, abstract_body_changed=True
),
)
await self._enqueue_parent_refresh(
msg,
target_uri or msg.uri,
l0_body_changed=write_result.abstract_body_changed,
)
processing_succeeded = True
finally:
await work.finish_processing(processing_succeeded)
failure = work.failure_result(dag_stats)
if failure is not None:
return failure
self._merge_request_stats(msg.telemetry_id, processed=1)
logger.info(f"Completed semantic generation for: {msg.uri}")
breaker.record_success()
await self._cleanup_local_artifact(msg)
return ProcessResult.success()
finally:
reset_root_observability_context(root_context_token)
except asyncio.CancelledError:
if work is not None:
await work.cancel()
raise
except Exception as e:
if isinstance(e, LockAcquisitionError):
execute_status = "requeued"
logger.warning(
"Lock error processing semantic message, re-enqueueing without "
"tripping API circuit breaker: %s",
e,
exc_info=True,
)
if msg is not None and work is not None:
result = await self._requeue_semantic_msg_after_error(
msg,
e,
work=work,
)
if result.error is not None:
execute_status = "error"
return result
return ProcessResult.failed(str(e))
kind = model_error_kind(e)
if kind is ModelErrorKind.INPUT_TOO_LARGE:
execute_status = "error"
logger.error(
f"Input too large processing semantic message, dropping: {e}",
exc_info=True,
)
if msg is not None:
self._merge_request_stats(msg.telemetry_id, error_count=1)
if msg is not None:
await self._cleanup_local_artifact(msg)
return ProcessResult.failed(str(e))
elif _is_permanent_storage_error(e) or kind in {
ModelErrorKind.MODEL_NOT_FOUND,
ModelErrorKind.UNSUPPORTED,
ModelErrorKind.INVALID_REQUEST,
ModelErrorKind.CONTENT_SAFETY,
}:
execute_status = "error"
logger.critical(
f"Permanent error processing semantic message, dropping: {e}",
exc_info=True,
)
if breaker is not None and kind is not None:
breaker.record_failure(e)
if msg is not None:
self._merge_request_stats(msg.telemetry_id, error_count=1)
if msg is not None:
await self._cleanup_local_artifact(msg)
return ProcessResult.failed(str(e))
else:
execute_status = "error"
if breaker is not None and kind is not None:
breaker.record_failure(e)
logger.error(
f"Error processing semantic message, not re-enqueueing: {e}",
exc_info=True,
)
if msg is not None:
self._merge_request_stats(msg.telemetry_id, error_count=1)
await self._cleanup_local_artifact(msg)
return ProcessResult.failed(str(e))
finally:
if msg is not None and execute_started_at is not None:
from openviking.metrics.datasources.resource import ResourceIngestionEventDataSource
ResourceIngestionEventDataSource.record_stage(
stage="semantic_queue_wait",
status=execute_status,
duration_seconds=queue_wait_ms / 1000.0,
account_id=msg.account_id,
)
ResourceIngestionEventDataSource.record_stage(
stage="semantic_execute",
status=execute_status,
duration_seconds=(time.perf_counter() - execute_started_at),
account_id=msg.account_id,
)
if work is not None:
await work.close()
async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> ProcessResult:
"""Release a queued semantic lock before cancelled work is ACKed."""
try:
import json
payload = data.get("data", data) if isinstance(data, dict) else data
if isinstance(payload, str):
payload = json.loads(payload)
msg = SemanticMsg.from_dict(payload)
except (TypeError, ValueError) as exc:
return ProcessResult.failed(str(exc))
await self._message_work(msg).cancel_queued()
await self._cleanup_local_artifact(msg)
return ProcessResult.cancelled()
async def _release_cancelled_semantic_lock(self, msg: SemanticMsg) -> None:
if msg.lock_handoff is not None:
lock_guard = PathLockGuard(get_viking_fs()._async_agfs)
try:
await lock_guard.adopt(msg.lock_handoff)
except Exception as exc:
logger.warning("Failed to release cancelled semantic lock: %s", exc)
finally:
await lock_guard.close()
async def _resolve_skill_semantic_lock(
self,
msg: SemanticMsg,
ctx: RequestContext,
caller_lock: Optional[Dict[str, Any]],
) -> SemanticLockScope:
scope = None
async def acquire():
nonlocal scope
viking_fs = get_viking_fs()
scope = await SemanticLockScope.resolve(
msg.lock_handoff,
caller_lock=caller_lock,
ctx=ctx,
fallback_path_factory=lambda: viking_fs._uri_to_path(msg.uri, ctx=ctx),
)
if scope.lock is None and await viking_fs.exists(msg.uri, ctx=ctx):
lease = await viking_fs._async_agfs.pathlock_acquire_tree(
viking_fs._uri_to_path(msg.uri, ctx=ctx)
)
scope = SemanticLockScope(lease, _owned=True)
return scope
try:
return await run_to_completion(acquire)
except asyncio.CancelledError:
# Acquiring/adopting a lease can itself outlive a cancelled await.
if scope is not None:
await run_to_completion(scope.close)
raise
def get_tree_stats(self) -> Optional["SemanticTreeStats"]:
return SemanticTreeExecutor.get_active_stats()
async def _process_memory_directory(
self,
msg: SemanticMsg,
ctx: Optional[RequestContext] = None,
lock: Optional[Dict[str, Any]] = None,
) -> None:
"""Process a memory directory with special handling.
For memory directories:
- Generate file summaries in bounded batches
- Enqueue each changed file for vectorization as soon as its summary is ready
- Retain only compact summaries for the directory overview
- Vectorize the generated abstract.md and overview.md
Args:
msg: The semantic message containing directory info and changes
"""
viking_fs = get_viking_fs()
dir_uri = msg.uri
ctx = ctx or self._default_ctx
llm_sem = asyncio.Semaphore(self.max_concurrent_llm)
try:
entries = await viking_fs.ls(dir_uri, node_limit=LS_ALL_NODES, ctx=ctx)
except Exception as e:
raise RuntimeError(f"Failed to list memory directory {dir_uri}: {e}") from e
file_paths: List[str] = []
for entry in entries:
name = entry.get("name", "")
if not name or name.startswith(".") or name in [".", ".."]:
continue
if not entry.get("isDir", False):
item_uri = VikingURI(dir_uri).join(name).uri
file_paths.append(item_uri)
file_paths.sort()
if not file_paths:
logger.info(f"No memory files found in {dir_uri}")
return
existing_summaries: Dict[str, str] = {}
if msg.changes:
try:
old_overview = await viking_fs.read_file(f"{dir_uri}/.overview.md", ctx=ctx)
if old_overview:
existing_summaries = self._parse_overview_md(body_for_preview(old_overview))
logger.info(
f"Parsed {len(existing_summaries)} existing summaries from overview.md"
)
except Exception as e:
logger.debug(f"No existing overview.md found for {dir_uri}: {e}")
changed_files: Set[str] = set()
if msg.changes:
changed_files = set(msg.changes.get("added", []) + msg.changes.get("modified", []))
deleted_files = set(msg.changes.get("deleted", []))
logger.info(
f"Processing memory directory {dir_uri} with changes: "
f"added={len(msg.changes.get('added', []))}, "
f"modified={len(msg.changes.get('modified', []))}, "
f"deleted={len(deleted_files)}"
)
pending_indices: List[Tuple[int, str]] = []
file_summaries: List[Optional[Dict[str, str]]] = [None] * len(file_paths)
paths_to_vectorize = changed_files if msg.changes else set(file_paths)
for idx, file_path in enumerate(file_paths):
file_name = file_path.split("/")[-1]
if file_path not in changed_files and file_name in existing_summaries:
file_summaries[idx] = {
"name": file_name,
"summary": existing_summaries[file_name],
}
logger.debug(f"Reused existing summary for {file_name}")
else:
pending_indices.append((idx, file_path))
if file_paths and not pending_indices:
try:
from openviking.metrics.datasources.cache import CacheEventDataSource
CacheEventDataSource.record_hit("L1")
except Exception:
pass
elif file_paths and pending_indices:
try:
from openviking.metrics.datasources.cache import CacheEventDataSource
if len(file_paths) > len(pending_indices):
CacheEventDataSource.record_hit("L1")
CacheEventDataSource.record_miss("L1")
except Exception:
pass
if pending_indices:
logger.info(
f"Generating summaries for {len(pending_indices)} changed files "
f"(reused {len(file_paths) - len(pending_indices)} cached)"
)
async def _gen(idx: int, file_path: str) -> None:
file_name = file_path.split("/")[-1]
try:
summary_dict = await self._generate_single_file_summary(
file_path, llm_sem=llm_sem, ctx=ctx
)
logger.debug(f"Generated summary for {file_name}")
except Exception as e:
logger.warning(f"Failed to generate summary for {file_path}: {e}")
summary_dict = {"name": file_name, "summary": ""}
if file_path in paths_to_vectorize and not msg.skip_vectorization:
await self._vectorize_single_file(
parent_uri=dir_uri,
context_type="memory",
file_path=file_path,
summary_dict=summary_dict,
ctx=ctx,
preserve_existing_created_at=True,
telemetry_id=msg.telemetry_id,
)
file_summaries[idx] = {
"name": str(summary_dict.get("name") or file_name),
"summary": str(summary_dict.get("summary") or ""),
}
batch_size = max(1, min(self.max_concurrent_llm, 10))
for batch_start in range(0, len(pending_indices), batch_size):
batch = pending_indices[batch_start : batch_start + batch_size]
logger.info(
f"[MemorySemantic] Processing batch {batch_start // batch_size + 1}/"
f"{(len(pending_indices) + batch_size - 1) // batch_size} "
f"({len(batch)} files)"
)
await asyncio.gather(*[_gen(i, fp) for i, fp in batch])
completed_summaries = [s for s in file_summaries if s is not None]
generated_content = await self._generate_overview(
dir_uri,
completed_summaries,
[],
llm_sem=llm_sem,
total_files=len(file_paths),
ctx=ctx,
)
overview, abstract = self._normalize_overview_generation(generated_content)
try:
wrote_semantics = await self._write_memory_directory_semantics(
msg=msg,
viking_fs=viking_fs,
dir_uri=dir_uri,
overview=overview,
abstract=abstract,
ctx=ctx,
lock=lock,
total_entries=len(file_paths),
sampled_entries=len(completed_summaries),
)
except LockAcquisitionError:
raise
except Exception as e:
raise RuntimeError(f"Failed to write abstract/overview for {dir_uri}: {e}") from e
if not wrote_semantics.wrote:
return
logger.info(f"Generated abstract.md and overview.md for {dir_uri}")
if msg.skip_vectorization:
logger.info(f"Skipping vectorization for {dir_uri} (requested via SemanticMsg)")
return
if not (wrote_semantics.overview_body_changed or wrote_semantics.abstract_body_changed):
logger.info(
"Skipping directory vectorization for %s (visible semantics unchanged)",
dir_uri,
)
return
await self._vectorize_directory(
uri=dir_uri,
context_type="memory",
abstract=abstract,
overview=overview,
ctx=ctx,
telemetry_id=msg.telemetry_id,
)
logger.info(f"Vectorized abstract.md and overview.md for {dir_uri}")
async def _write_memory_directory_semantics(
self,
*,
msg: SemanticMsg,
viking_fs: Any,
dir_uri: str,
overview: str,
abstract: str,
ctx: Optional[RequestContext],
lock: Optional[Dict[str, Any]] = None,
total_entries: int = 0,
sampled_entries: int = 0,
) -> AbstractOverviewWriteResult:
return await write_abstract_overview(
viking_fs=viking_fs,
dir_uri=dir_uri,
overview=overview,
abstract=abstract,
ctx=ctx,
is_stale=lambda: is_semantic_msg_stale(msg),
metadata={
**({"source": msg.source} if msg.source else {}),
"generated_by": {
"component": "SemanticProcessor",
"trigger": msg.generation_trigger,
},
"freshness": freshness_metadata(total_entries, sampled_entries),
},
lock=lock,
log_prefix="[MemorySemantic]",
)
async def _sync_topdown_recursive(
self,
root_uri: str,
target_uri: str,
ctx: Optional[RequestContext] = None,
file_change_status: Optional[Dict[str, bool]] = None,
lock: Optional[Dict[str, Any]] = None,
) -> SyncDiff:
"""Merge a temp/staging source tree into the target via VikingFS.sync_tree.
The pure filesystem diff-and-move is delegated to
:py:meth:`VikingFS.sync_tree`; this wrapper carries over parser
sidecars (``.image_mappings.json``) and rewrites markdown image
references once the visible files have been moved into place.
Hidden entries are skipped by ``sync_tree``, which is why the
sidecar handling lives here rather than in the FS layer.
"""
viking_fs = get_viking_fs()
if not await viking_fs.exists(root_uri, ctx=ctx):
raise FileNotFoundError(
f"Semantic source no longer exists; refusing to sync into {target_uri}: {root_uri}"
)
target_exists = await viking_fs.exists(target_uri, ctx=ctx)
diff = await viking_fs.sync_tree(
root_uri,
target_uri,
ctx=ctx,
file_change_status=file_change_status,
lease_ref=lock,
delete_temp_after=False,
)
if not target_exists:
# The whole temp tree (including the hidden .image_mappings.json
# sidecar) was moved into the target; rewrite local image paths now.
await self._rewrite_target_image_uris(root_uri, target_uri, ctx=ctx, lock=lock)
return diff
# sync_tree skips hidden files, so the .image_mappings.json sidecar is
# still at the temp root. Carry it over and rewrite the synced markdown
# before the temp tree is deleted below.
await self._rewrite_target_image_uris(root_uri, target_uri, ctx=ctx, lock=lock)
try:
await viking_fs.delete_temp(root_uri, ctx=ctx)
except Exception as e:
logger.error(f"[SyncDiff] Failed to delete root directory {root_uri}: {e}")
return diff
async def _rewrite_target_image_uris(
self,
root_uri: str,
target_uri: str,
ctx: Optional[RequestContext] = None,
lock: Optional[Dict[str, Any]] = None,
) -> None:
"""Rewrite local image refs in the target after a temp-to-target sync.
``_sync_topdown_recursive`` MOVES the visible files into the target and
skips hidden ones, so afterwards the temp tree holds only the
``.image_mappings.json`` sidecars written by the parser (one per
document root, possibly nested for directory ingests). Discovery is
therefore driven by the markdown files already synced into the TARGET:
their ancestor directories, mirrored back onto the temp tree, are where
sidecars can live. Carry each one over (when missing) so
:func:`rewrite_image_uris` can resolve local image paths against the
images that were synced into the final target.
"""
viking_fs = get_viking_fs()
root_prefix = root_uri.rstrip("/")
target_prefix = target_uri.rstrip("/")
mapping_name = IMAGE_MAPPINGS_FILENAME
if root_prefix != target_prefix:
try:
glob_result = await viking_fs.glob("**/*.md", uri=target_prefix, ctx=ctx)
target_md_uris = glob_result.get("matches", [])
except Exception:
target_md_uris = []
# Ancestor dirs of the target md files, as paths relative to the
# target root — the candidate sidecar locations on both trees.
candidate_rels = set()
for md_uri in target_md_uris:
d = md_uri.rsplit("/", 1)[0]
while d == target_prefix or d.startswith(target_prefix + "/"):
candidate_rels.add(d[len(target_prefix) :].lstrip("/"))
if d == target_prefix:
break
d = d.rsplit("/", 1)[0]
for rel in candidate_rels:
src_mapping = (
f"{root_prefix}/{rel}/{mapping_name}"
if rel
else f"{root_prefix}/{mapping_name}"
)
target_mapping = (
f"{target_prefix}/{rel}/{mapping_name}"
if rel
else f"{target_prefix}/{mapping_name}"
)
try:
await viking_fs.stat(target_mapping, ctx=ctx, skip_count=True)
continue # already carried over
except Exception:
pass
try:
mapping_content = await viking_fs.read_file(src_mapping, ctx=ctx)
except Exception:
continue # no sidecar at this level
try:
# TODO: This must be optimized once pathlock is pushed down into ragfs.
await viking_fs.write_file(
target_mapping,
mapping_content,
ctx=ctx,
lease_ref=lock,
)
except Exception:
# Target subtree may not exist (doc removed in sync); skip.
pass
try:
await rewrite_image_uris(target_uri, ctx=ctx, lease_ref=lock)
except Exception as e:
logger.error(f"[SyncDiff] Failed to rewrite image URIs for {target_uri}: {e}")
async def _generate_text_summary(
self,
file_path: str,
file_name: str,
llm_sem: asyncio.Semaphore,
ctx: Optional[RequestContext] = None,
file_content: Optional[bytes] = None,
materialize_content: bool = False,
) -> Dict[str, Any]:
"""Generate summary for a single text file (code, documentation, or other text)."""
viking_fs = get_viking_fs()
active_ctx = ctx or self._default_ctx
content = (
file_content
if file_content is not None
else await viking_fs.read_file(file_path, ctx=active_ctx)
)
if isinstance(content, bytes):
from openviking.utils.embedding_utils import _decode_text_bytes
content = _decode_text_bytes(content)
def result(summary: str) -> Dict[str, Any]:
return {"name": file_name, "summary": summary}
config = get_openviking_config()
# Limit content length
max_chars = config.semantic.max_file_content_chars
if len(content) > max_chars:
content = content[:max_chars] + "\n...(truncated)"
# Detect file type and select appropriate prompt
file_type = self._detect_file_type(file_name)
if file_type == FILE_TYPE_CODE:
from openviking.parse.parsers.code.ast import extract_skeleton_result
extraction = extract_skeleton_result(file_name, content)
if extraction.text:
skeleton_text = extraction.text
max_skeleton_chars = config.semantic.max_skeleton_chars
if len(skeleton_text) > max_skeleton_chars:
skeleton_text = skeleton_text[:max_skeleton_chars]
return result(skeleton_text)
prompt_id = "semantic.code_summary"
elif file_type == FILE_TYPE_DOCUMENTATION:
prompt_id = "semantic.document_summary"
else:
prompt_id = "semantic.file_summary"
vlm = await self._get_vlm_config(active_ctx)
if not vlm.is_available():
if file_type == FILE_TYPE_CODE:
logger.warning("VLM not available for code summary fallback: %s", file_path)
else:
logger.warning("VLM not available, using empty summary")
return result("")
from openviking.session.memory.utils.language import resolve_output_language
output_language = resolve_output_language(content, config=config)
prompt = render_prompt(
prompt_id,
{"file_name": file_name, "content": content, "output_language": output_language},
)
async with llm_sem:
with bind_telemetry_stage("semantic_execute"):
summary = await vlm.get_completion_async(prompt)
return result(summary.strip())
async def _generate_single_file_summary(
self,
file_path: str,
llm_sem: Optional[asyncio.Semaphore] = None,
ctx: Optional[RequestContext] = None,
file_content: Optional[bytes] = None,
) -> Dict[str, Any]:
"""Generate summary for a single file.
Args:
file_path: File path
Returns:
{"name": file_name, "summary": summary_content}
"""
file_name = file_path.split("/")[-1]
llm_sem = llm_sem or asyncio.Semaphore(self.max_concurrent_llm)
media_type = get_media_type(file_name, None)
if file_name.lower().endswith(".ts"):
try:
prefix = (
file_content[:MPEG_TS_PROBE_BYTES]
if file_content is not None
else await get_viking_fs().read(
file_path,
offset=0,
size=MPEG_TS_PROBE_BYTES,
ctx=ctx,
)
)
except Exception:
prefix = None
media_type = get_media_type(file_name, None, content=prefix)
if media_type in {"image", "audio", "video"}:
vlm = await self._get_vlm_config(ctx or self._default_ctx)
if media_type == "image":
return await generate_image_summary(
file_path,
file_name,
llm_sem,
ctx=ctx,
vlm=vlm,
)
if media_type == "audio":
return await generate_audio_summary(
file_path,
file_name,
llm_sem,
ctx=ctx,
vlm=vlm,
)
return await generate_video_summary(
file_path,
file_name,
llm_sem,
ctx=ctx,
vlm=vlm,
)
return await self._generate_text_summary(
file_path, file_name, llm_sem, ctx=ctx, file_content=file_content
)
def _child_summary_line(
self,
dir_uri: str,
idx: int,
item: Dict[str, str],
link_map: Dict[str, str],
) -> str:
"""Render a subdirectory summary line with a collision-free link placeholder."""
placeholder = f"viking://input_sample_c{idx}"
link_map[placeholder] = self._markdown_link_target(dir_uri, item["name"])
return f"- {item['name']}/ (link: {placeholder}): {item['abstract']}"
@staticmethod
def _markdown_link_target(dir_uri: str, entry_name: str) -> str:
"""Build a Markdown-safe target without changing the stored Viking URI."""
entry_uri = VikingURI(dir_uri).join(entry_name).uri
return markdown_safe_viking_uri(entry_uri)
def _replace_link_references(self, generated_content: str, link_map: Dict[str, str]) -> str:
"""Resolve link placeholders (viking://input_sample_fN / cN) to real URIs.
The model is fed compact, collision-free placeholders and asked to emit
Markdown links against them; here we map each placeholder back to the
entry's real URI. Unknown placeholders are left untouched.
"""
def replace_link(match):
return link_map.get(match.group(0), match.group(0))
return re.sub(r"viking://input_sample_[fc]\d+", replace_link, generated_content)
def _truncate_generated_text(self, text: str, max_chars: int) -> str:
if max_chars <= 0 or len(text) <= max_chars:
return text
if max_chars <= 3:
return text[:max_chars]
first_sentence_end = None
last_sentence_end_within_limit = None
for sentence_end_match in re.finditer(r"\.(?!\d)(?=\s|$)|[!?](?=\s|$)|[。?!]", text):
sentence_end = sentence_end_match.end()
if first_sentence_end is None:
first_sentence_end = sentence_end
if sentence_end <= max_chars:
last_sentence_end_within_limit = sentence_end
elif last_sentence_end_within_limit is not None:
break
if last_sentence_end_within_limit is not None:
return text[:last_sentence_end_within_limit].strip()
if first_sentence_end is not None:
return text[:first_sentence_end].strip()
candidate = text[: max_chars - 3].rstrip()
word_boundary = candidate.rfind(" ")
if word_boundary > 0:
return candidate[:word_boundary].rstrip() + "..."
return candidate + "..."
def _extract_abstract_from_overview(self, overview_content: str) -> str:
"""Extract an abstract from the Markdown overview brief description."""
overview_content = body_for_preview(overview_content)
lines = overview_content.split("\n")
# Skip header lines (starting with #)
content_lines = []
in_header = True
for line in lines:
if line.strip() == "---":
continue
if in_header and line.startswith("#"):
continue
elif in_header and line.strip():
in_header = False
if not in_header:
# Stop at first ##
if line.startswith("##"):
break
if line.strip():
content_lines.append(line.strip())
return "\n".join(content_lines).strip()
def _normalize_overview_generation(self, generated_content: str) -> Tuple[str, str]:
"""Convert raw Markdown overview output into final L1 overview and L0 abstract."""
overview = body_for_preview(generated_content)
abstract = self._extract_abstract_from_overview(overview)
return self._enforce_size_limits(overview, abstract)
def _enforce_size_limits(self, overview: str, abstract: str) -> Tuple[str, str]:
"""Enforce max size limits on overview and abstract."""
semantic = get_openviking_config().semantic
if len(overview) > semantic.overview_max_chars:
overview = self._truncate_generated_text(overview, semantic.overview_max_chars)
if len(abstract) > semantic.abstract_max_chars:
abstract = self._truncate_generated_text(abstract, semantic.abstract_max_chars)
return overview, abstract
@classmethod
def _parse_overview_md(cls, overview_content: str) -> Dict[str, str]:
"""Parse overview.md and extract direct-file summaries."""
return parse_overview_file_summaries(overview_content)
async def _generate_overview(
self,
dir_uri: str,
file_summaries: List[Dict[str, str]],
children_abstracts: List[Dict[str, str]],
llm_sem: Optional[asyncio.Semaphore] = None,
total_files: Optional[int] = None,
total_children: Optional[int] = None,
ctx: Optional[RequestContext] = None,
) -> str:
"""Generate raw directory overview model output.
For small directories, generates a single overview from all file summaries.
For large directories that would exceed the prompt budget, splits file
summaries into batches, generates a partial overview per batch, then
merges the partials into a final overview.
Args:
dir_uri: Directory URI
file_summaries: File summary list (only entries participating in
this aggregation, i.e. already sampled when applicable)
children_abstracts: Subdirectory summary list (sampled when applicable)
total_files: Total direct files in the directory (may exceed
``len(file_summaries)`` when the directory was sampled)
total_children: Total direct subdirectories (may exceed
``len(children_abstracts)`` when sampled)
Returns:
Markdown overview content generated by the model.
"""
config = get_openviking_config()
active_ctx = ctx or self._default_ctx
vlm = await self._get_vlm_config(active_ctx)
semantic = config.semantic
if not vlm.is_available():
logger.warning("VLM not available, using default overview")
return f"# {dir_uri.split('/')[-1]}\n\n[Directory overview is not ready]"
from openviking.session.memory.utils.language import resolve_output_language
# Build coverage statement so the model knows the directory scale
# and whether the provided summaries represent all entries or a sample.
total_files = len(file_summaries) if total_files is None else total_files
total_children = len(children_abstracts) if total_children is None else total_children
total_direct = total_files + total_children
provided = len(file_summaries) + len(children_abstracts)
if total_direct <= provided:
directory_coverage = (
f"Total direct entries: {total_direct}. "
"All direct entries are represented in the summaries below."
)
else:
directory_coverage = (
f"Total direct entries: {total_direct} "
f"({total_files} files, {total_children} subdirectories).\n"
f"Summaries provided for this aggregation: {provided}.\n"
f"Direct entries not individually shown: {total_direct - provided}.\n"
"Coverage: sampled. The summaries below are representative "
"entries; generalize cautiously and do not treat them as exhaustive."
)
# Build link placeholder mapping and summary string.
# The model is fed collision-free placeholders (viking://input_sample_fN)
# and asked to emit Markdown links; placeholders are resolved to real
# URIs in post-processing. This avoids the fragility of bare [N] indices
# and of asking the model to reproduce long/CJK URIs verbatim.
link_map: Dict[str, str] = {}
file_summaries_lines = []
for idx, item in enumerate(file_summaries, 1):
placeholder = f"viking://input_sample_f{idx}"
link_map[placeholder] = self._markdown_link_target(dir_uri, item["name"])
file_summaries_lines.append(
f"- {item['name']} (link: {placeholder}): {item['summary']}"
)
file_summaries_str = "\n".join(file_summaries_lines) if file_summaries_lines else "None"
# Build subdirectory summary string
children_abstracts_str = (
"\n".join(
self._child_summary_line(dir_uri, idx, item, link_map)
for idx, item in enumerate(children_abstracts, 1)
)
if children_abstracts
else "None"
)
language_source_parts = []
if file_summaries:
language_source_parts.append(file_summaries_str)
if children_abstracts:
language_source_parts.append(children_abstracts_str)
if not language_source_parts:
language_source_parts.append(dir_uri.split("/")[-1])
output_language = resolve_output_language("\n".join(language_source_parts), config=config)
# Budget guard: check if prompt would be oversized
estimated_size = len(file_summaries_str) + len(children_abstracts_str)
over_budget = estimated_size > semantic.max_overview_prompt_chars
entry_count = len(file_summaries) + len(children_abstracts)
many_entries = entry_count > semantic.overview_batch_size
if over_budget and many_entries:
# Many entries, oversized prompt → batch and merge
logger.info(
f"Overview prompt for {dir_uri} exceeds budget "
f"({estimated_size} chars, {len(file_summaries)} files, "
f"{len(children_abstracts)} subdirectories). "
f"Splitting into batches of {semantic.overview_batch_size}."
)
overview = await self._batched_generate_overview(
dir_uri,
file_summaries,
children_abstracts,
link_map,
llm_sem=llm_sem,
output_language=output_language,
directory_coverage=directory_coverage,
vlm=vlm,
)
elif over_budget:
# Few files but long summaries → truncate summaries to fit budget
logger.info(
f"Overview prompt for {dir_uri} exceeds budget "
f"({estimated_size} chars) with {len(file_summaries)} files. "
f"Truncating summaries to fit."
)
budget = semantic.max_overview_prompt_chars
budget -= len(children_abstracts_str)
per_file = max(100, budget // max(len(file_summaries), 1))
truncated_lines = []
for idx, item in enumerate(file_summaries, 1):
summary = item["summary"][:per_file]
truncated_lines.append(
f"- {item['name']} (link: viking://input_sample_f{idx}): {summary}"
)
file_summaries_str = "\n".join(truncated_lines)
overview = await self._single_generate_overview(
dir_uri,
file_summaries_str,
children_abstracts_str,
link_map,
output_language=output_language,
directory_coverage=directory_coverage,
vlm=vlm,
)
else:
overview = await self._single_generate_overview(
dir_uri,
file_summaries_str,
children_abstracts_str,
link_map,
output_language=output_language,
directory_coverage=directory_coverage,
vlm=vlm,
)
return overview
async def _single_generate_overview(
self,
dir_uri: str,
file_summaries_str: str,
children_abstracts_str: str,
link_map: Dict[str, str],
output_language: str = "en",
directory_coverage: str = "",
*,
vlm: "VLMHandle",
) -> str:
"""Generate overview from a single prompt (small directories)."""
try:
prompt = render_prompt(
"semantic.overview_generation",
{
"dir_name": dir_uri.split("/")[-1],
"file_summaries": file_summaries_str,
"children_abstracts": children_abstracts_str,
"output_language": output_language,
"directory_coverage": directory_coverage,
},
)
with bind_telemetry_stage("semantic_execute"):
overview = await vlm.get_completion_async(prompt)
overview = self._replace_link_references(overview, link_map)
return overview.strip()
except Exception as e:
logger.error(
f"Failed to generate overview for {dir_uri}: {e}",
exc_info=True,
)
return f"# {dir_uri.split('/')[-1]}\n\n[Directory overview is not generated]"
async def _batched_generate_overview(
self,
dir_uri: str,
file_summaries: List[Dict[str, str]],
children_abstracts: List[Dict[str, str]],
link_map: Dict[str, str],
llm_sem: Optional[asyncio.Semaphore] = None,
output_language: str = "en",
directory_coverage: str = "",
*,
vlm: "VLMHandle",
) -> str:
"""Generate overview by batching file and subdirectory summaries.
Splits both input kinds into batches, generates a partial overview per
batch, then merges the partials without repeating the raw inputs.
"""
config = get_openviking_config()
semantic = config.semantic
batch_size = semantic.overview_batch_size
dir_name = dir_uri.split("/")[-1]
work_items = [("file", index, item) for index, item in enumerate(file_summaries, 1)] + [
("child", index, item) for index, item in enumerate(children_abstracts, 1)
]
batches = [work_items[i : i + batch_size] for i in range(0, len(work_items), batch_size)]
logger.info(f"Generating overview for {dir_uri} in {len(batches)} batches")
# Generate partial overviews concurrently using global link placeholders.
if llm_sem is None:
llm_sem = asyncio.Semaphore(self.max_concurrent_llm)
partial_overviews = [None] * len(batches)
batch_prompts: List[Tuple[int, str, Dict[str, str]]] = []
for batch_idx, batch in enumerate(batches):
batch_lines = []
child_lines = []
batch_link_map: Dict[str, str] = {}
for entry_kind, global_idx, item in batch:
assert global_idx is not None
if entry_kind == "file":
placeholder = f"viking://input_sample_f{global_idx}"
batch_link_map[placeholder] = link_map.get(
placeholder, self._markdown_link_target(dir_uri, item["name"])
)
batch_lines.append(f"- {item['name']} (link: {placeholder}): {item['summary']}")
else:
child_lines.append(
self._child_summary_line(dir_uri, global_idx, item, batch_link_map)
)
prompt = render_prompt(
"semantic.overview_generation",
{
"dir_name": dir_name,
"file_summaries": "\n".join(batch_lines) or "None",
"children_abstracts": "\n".join(child_lines) or "None",
"output_language": output_language,
"directory_coverage": directory_coverage,
},
)
batch_prompts.append((batch_idx, prompt, batch_link_map))
async def _run_batch(batch_idx: int, prompt: str, batch_link_map: Dict[str, str]) -> None:
try:
async with llm_sem:
with bind_telemetry_stage("semantic_execute"):
partial = await vlm.get_completion_async(prompt)
partial = self._replace_link_references(partial, batch_link_map)
partial_overviews[batch_idx] = partial.strip()
except Exception as e:
logger.warning(
f"Failed to generate partial overview batch "
f"{batch_idx + 1}/{len(batches)} for {dir_uri}: {e}"
)
await asyncio.gather(*[_run_batch(*bp) for bp in batch_prompts])
partial_overviews = [p for p in partial_overviews if p is not None]
if not partial_overviews:
return f"# {dir_name}\n\n[Directory overview is not generated]"
# If only one batch succeeded, use it directly
if len(partial_overviews) == 1:
return partial_overviews[0]
# Merge partials only; each child abstract is already represented once.
# Partials already contain real URIs, but the shared generation prompt can
# make the merge model emit placeholders again. Resolve the merged output
# with the complete map before persisting it.
combined = "\n\n---\n\n".join(partial_overviews)
try:
prompt = render_prompt(
"semantic.overview_generation",
{
"dir_name": dir_name,
"file_summaries": combined,
"children_abstracts": "None",
"output_language": output_language,
"directory_coverage": directory_coverage,
},
)
with bind_telemetry_stage("semantic_execute"):
overview = await vlm.get_completion_async(prompt)
overview = self._replace_link_references(overview, link_map)
return overview.strip()
except Exception as e:
logger.error(
f"Failed to merge partial overviews for {dir_uri}: {e}",
exc_info=True,
)
return partial_overviews[0]
async def _skill_root_semantics(
self,
uri: str,
*,
ctx: RequestContext,
regenerate: bool = False,
lock: Optional[Dict[str, Any]] = None,
skill_content: str | bytes | None = None,
) -> Tuple[str, str]:
"""Keep the package root tied only to its SKILL.md definition."""
viking_fs = get_viking_fs()
if (
not regenerate
and await viking_fs.exists(f"{uri}/.abstract.md", ctx=ctx)
and await viking_fs.exists(f"{uri}/.overview.md", ctx=ctx)
):
abstract = body_for_preview(await viking_fs.read_file(f"{uri}/.abstract.md", ctx=ctx))
overview = body_for_preview(await viking_fs.read_file(f"{uri}/.overview.md", ctx=ctx))
if abstract and overview:
return overview, abstract
from openviking.core.skill_loader import SkillLoader
from openviking.utils.skill_processor import SkillProcessor
content = (
skill_content
if skill_content is not None
else await viking_fs.read_file(f"{uri}/SKILL.md", ctx=ctx)
)
if isinstance(content, bytes):
content = content.decode("utf-8")
definition = SkillLoader.parse(content)
processor = SkillProcessor(vikingdb=None, vlm_resolver=self._vlm_resolver)
abstract = processor._build_skill_abstract(definition)
overview = await processor._generate_overview(
definition,
await self._get_vlm_config(ctx),
)
await run_to_completion(
lambda: write_abstract_overview(
viking_fs=viking_fs,
dir_uri=uri,
abstract=abstract,
overview=overview,
ctx=ctx,
lock=lock,
is_stale=lambda: False,
metadata={
"generated_by": {"component": "SkillProcessor", "trigger": "skill_refresh"}
},
)
)
return overview, abstract
async def _vectorize_directory(
self,
uri: str,
context_type: str,
abstract: str,
overview: str,
ctx: Optional[RequestContext] = None,
ingest_options: IngestOptions | None = None,
skill_source_path: str = "",
scalar_overrides: Optional[Dict[int, Dict[str, Any]]] = None,
actions: Optional[Dict[int, str]] = None,
field_patches: Optional[Dict[int, FieldPatch]] = None,
include_abstract: bool = True,
include_overview: bool = True,
telemetry_id: str | None = None,
md5s: Optional[Dict[int, str]] = None,
) -> set[int]:
"""Create directory Context and enqueue to EmbeddingQueue."""
from openviking.utils.embedding_utils import vectorize_directory_meta
active_ctx = ctx or self._default_ctx
skill_meta = None
if context_type == "skill" and classify_uri(uri).is_skill_root:
import yaml
parsed = yaml.safe_load(body_for_preview(abstract))
if isinstance(parsed, dict):
skill_meta = {
key: parsed.get(key, [] if key in {"tags", "allowed_tools"} else "")
for key in ("name", "description", "tags", "allowed_tools")
}
skill_meta["source_path"] = skill_source_path
return await vectorize_directory_meta(
uri=uri,
abstract=abstract,
overview=overview,
context_type=context_type,
ctx=active_ctx,
ingest_options=ingest_options,
content_is_body=context_type == "skill",
**({"meta": skill_meta} if skill_meta is not None else {}),
scalar_overrides=scalar_overrides,
actions=actions,
field_patches=field_patches,
include_abstract=include_abstract,
include_overview=include_overview,
telemetry_id=telemetry_id,
md5s=md5s,
)
async def _load_transfer_file_summaries(
self,
file_paths: List[str],
ctx: Optional[RequestContext] = None,
) -> Dict[str, str]:
"""Load copied/moved file summaries from their existing target L2 vectors."""
if not file_paths:
return {}
viking_fs = get_viking_fs()
vector_store = viking_fs._get_vector_store()
if vector_store is None:
return {}
active_ctx = ctx or self._default_ctx
return await vector_store.get_l2_abstracts_by_uris(file_paths, ctx=active_ctx)
async def _update_vector_fields(
self,
*,
record_id: str,
uri: str,
level: int,
field_patch: FieldPatch,
ctx: RequestContext,
telemetry_id: str | None = None,
) -> bool:
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
from openviking.telemetry import get_current_telemetry
from openviking.utils.embedding_utils import _enqueue_embedding_message
embedding_msg = EmbeddingMsg.for_update_fields(
record_id=record_id,
field_patch=field_patch,
context_data={
"uri": uri,
"level": level,
"account_id": ctx.account_id,
"owner_user_id": ctx.user.user_id,
},
telemetry_id=(
get_current_telemetry().telemetry_id if telemetry_id is None else telemetry_id
),
)
queue_manager = get_queue_manager()
embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING, allow_create=True)
return await _enqueue_embedding_message(
embedding_queue,
embedding_msg,
)
async def _vectorize_single_file(
self,
parent_uri: str,
context_type: str,
file_path: str,
summary_dict: Dict[str, str],
ctx: Optional[RequestContext] = None,
use_summary: bool = False,
preserve_existing_created_at: bool = False,
ingest_options: IngestOptions | None = None,
file_md5: Optional[str] = None,
file_content: Optional[bytes] = None,
materialize_content: bool = False,
scalar_override: Optional[Dict[str, Any]] = None,
field_patch: FieldPatch | None = None,
action: str = "merge",
telemetry_id: str | None = None,
) -> bool:
"""Vectorize a single file using its content or summary."""
from openviking.utils.embedding_utils import vectorize_file
active_ctx = ctx or self._default_ctx
return await vectorize_file(
file_path=file_path,
summary_dict=summary_dict,
parent_uri=parent_uri,
context_type=context_type,
ctx=active_ctx,
use_summary=use_summary,
preserve_existing_created_at=preserve_existing_created_at,
ingest_options=ingest_options,
file_md5=file_md5,
file_content=file_content,
materialize_content=materialize_content,
scalar_override=scalar_override,
field_patch=field_patch,
action=action,
telemetry_id=telemetry_id,
)