Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: terminate semantic work on AGFS not-a-directory errors (#4934)
Matt Van Hornb0c5e8571113
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex c43fb9036..7fb73d552 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -33,6 +33,7 @@ from openviking.parse.parsers.media.utils import (get_media_type,)from openviking.prompts import render_prompt+from openviking.pyagfs.exceptions import AGFSNotADirectoryErrorfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.service.task_work_index import detach_task_context@@ -629,10 +630,12 @@ class SemanticProcessor(DequeueHandlerBase):return ProcessResult.failed(str(e))elif error_class == ERROR_CLASS_PERMANENT:logger.critical(- f"Permanent API error processing semantic message, dropping: {e}",+ f"Permanent error processing semantic message, dropping: {e}",exc_info=True,)- self._circuit_breaker.record_failure(e)+ # A malformed filesystem target does not indicate an API outage.+ if not any(isinstance(exc, AGFSNotADirectoryError) for exc in (e, e.__cause__)):+ self._circuit_breaker.record_failure(e)if msg is not None:self._merge_request_stats(msg.telemetry_id, error_count=1)get_request_wait_tracker().mark_semantic_failed(diff --git a/openviking/utils/model_retry.py b/openviking/utils/model_retry.pyindex 7f12fbc9b..9b6d7f6e2 100644--- a/openviking/utils/model_retry.py+++ b/openviking/utils/model_retry.py@@ -8,6 +8,7 @@ import threadingimport timefrom typing import Awaitable, Callable, TypeVar+from openviking.pyagfs.exceptions import AGFSNotADirectoryErrorfrom openviking.utils.exceptions import AllCredentialsFailedErrorlogger = logging.getLogger(__name__)@@ -74,6 +75,7 @@ def extract_metric_error_code(error: BaseException) -> str:return "connection_error"return "unknown"+INPUT_TOO_LARGE_PATTERNS = ("413","payload too large",@@ -130,7 +132,13 @@ QUOTA_EXCEEDED_PATTERNS = ("usage quota",)-_PERMANENT_IO_ERRORS = (FileNotFoundError, PermissionError, IsADirectoryError, NotADirectoryError)+_PERMANENT_IO_ERRORS = (+ FileNotFoundError,+ PermissionError,+ IsADirectoryError,+ NotADirectoryError,+ AGFSNotADirectoryError,+)TRANSIENT_API_ERROR_PATTERNS = ("429",diff --git a/tests/storage/test_semantic_processor_permanent_storage_error.py b/tests/storage/test_semantic_processor_permanent_storage_error.pynew file mode 100644index 000000000..e78f603c2--- /dev/null+++ b/tests/storage/test_semantic_processor_permanent_storage_error.py@@ -0,0 +1,290 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Permanent storage failures must finish semantic work without blocking the API."""++import asyncio+import json+from types import SimpleNamespace+from unittest.mock import AsyncMock, Mock, call+from uuid import uuid4++import pytest++from openviking.pyagfs.exceptions import AGFSNotADirectoryError+from openviking.service.task_work_index import TaskWorkIndex+from openviking.storage.errors import LockAcquisitionError+from openviking.storage.queuefs.named_queue import NamedQueue+from openviking.storage.queuefs.semantic_dag import DagStats+from openviking.storage.queuefs.semantic_msg import SemanticMsg+from openviking.storage.queuefs.semantic_processor import SemanticProcessor+from openviking.storage.viking_fs import SyncDiff+from openviking.telemetry.request_wait_tracker import get_request_wait_tracker+from openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen+++def semantic_env(monkeypatch):+ lease = {"lease_ref": "semantic-lease", "owner_id": "consumer", "owned": True}+ pathlock = SimpleNamespace(+ pathlock_adopt=AsyncMock(return_value=lease),+ pathlock_as_borrowed=AsyncMock(return_value={**lease, "owned": False}),+ pathlock_release=AsyncMock(),+ )+ fs = SimpleNamespace(_async_agfs=pathlock, exists=AsyncMock(return_value=True))+ monkeypatch.setattr("openviking.storage.queuefs.semantic_processor.get_viking_fs", lambda: fs)+ monkeypatch.setattr("openviking.storage.queuefs.semantic_lock.get_viking_fs", lambda: fs)+ dag = SimpleNamespace(run=AsyncMock(), get_stats=Mock(return_value=DagStats()), stale=False)+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_processor.SemanticDagExecutor", Mock(return_value=dag)+ )+ retry_queue = SimpleNamespace(enqueue=AsyncMock())+ manager = SimpleNamespace(SEMANTIC="Semantic", get_queue=lambda name: retry_queue)+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: manager)+ processor = SemanticProcessor()+ processor._sync_topdown_recursive = AsyncMock(return_value=SyncDiff())+ success, requeue, error = Mock(), Mock(), Mock()+ processor.set_callbacks(success, requeue, error)+ tracker = get_request_wait_tracker()+ telemetry_ids = []++ def message(**kwargs):+ msg = SemanticMsg(+ uri="viking://resources/notes.txt",+ context_type="resource",+ telemetry_id=str(uuid4()),+ propagate_to_parent=False,+ **kwargs,+ )+ telemetry_ids.append(msg.telemetry_id)+ tracker.register_request(msg.telemetry_id)+ tracker.register_semantic_root(msg.telemetry_id, msg.id)+ return msg++ yield SimpleNamespace(+ processor=processor,+ pathlock=pathlock,+ lease=lease,+ dag=dag,+ retry_queue=retry_queue,+ success=success,+ requeue=requeue,+ error=error,+ tracker=tracker,+ message=message,+ )+ for telemetry_id in telemetry_ids:+ tracker.cleanup(telemetry_id)+++def _storage_error(wrapped):+ error = AGFSNotADirectoryError("timeout at /arbitrary/notes.txt")+ if wrapped:+ wrapper = RuntimeError("semantic storage operation failed")+ wrapper.__cause__ = error+ return wrapper+ return error+++async def _assert_request_failed(env, msg, error):+ await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)+ assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"] == {+ "processed": 0,+ "requeue_count": 0,+ "error_count": 1,+ "errors": [{"message": str(error)}],+ }++[email protected]("wrapped", [False, True])[email protected]("boundary", ["sync", "dag"])[email protected]("owned", [False, True])+async def test_storage_error_finishes_request_and_respects_lock_ownership(+ semantic_env, wrapped, boundary, owned+):+ env = semantic_env+ msg = env.message(+ target_uri="viking://resources/existing.txt" if boundary == "sync" else "",+ lock_handoff={"owner_id": "producer"} if owned else None,+ )+ error = _storage_error(wrapped)+ operation = env.processor._sync_topdown_recursive if boundary == "sync" else env.dag.run+ operation.side_effect = error+ data = msg.to_dict()++ await asyncio.wait_for(+ env.processor.on_dequeue(data, lock=None if owned else env.lease), timeout=1+ )++ operation.assert_awaited_once()+ env.error.assert_called_once_with(str(error), data)+ env.success.assert_not_called()+ env.requeue.assert_not_called()+ env.retry_queue.enqueue.assert_not_awaited()+ await _assert_request_failed(env, msg, error)+ env.processor._circuit_breaker.check()+ if owned:+ env.pathlock.pathlock_adopt.assert_awaited_once_with(msg.lock_handoff)+ env.pathlock.pathlock_release.assert_awaited_once_with(env.lease)+ else:+ env.pathlock.pathlock_as_borrowed.assert_awaited_once_with(env.lease)+ env.pathlock.pathlock_release.assert_not_awaited()++[email protected]("wrapped", [False, True])[email protected]("task_owned", [False, True])+async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrapped, task_owned):+ env = semantic_env+ msg = env.message(lock_handoff={"owner_id": "producer"})+ error = _storage_error(wrapped)+ env.dag.run.side_effect = error+ backend = SimpleNamespace(mkdir=AsyncMock(), write=AsyncMock(return_value="queue-message"))+ monkeypatch.setattr(+ "openviking.storage.queuefs.named_queue.AsyncAGFSClient", lambda agfs: backend+ )+ index = TaskWorkIndex()+ finalize = AsyncMock()+ index.set_callbacks(+ finalize_before_ack=finalize, is_cancellation_requested=lambda task_id: False+ )+ queue = NamedQueue(+ object(), "/queue", "Semantic", dequeue_handler=env.processor, task_work_index=index+ )+ payload = msg.to_dict()+ if task_owned:+ payload["task_id"] = "ingestion-task"+ await queue.enqueue(payload)+ queued_payload = backend.write.call_args.args[1].decode()+ backend.read = AsyncMock(+ side_effect=[json.dumps({"id": "queue-message", "data": queued_payload}).encode(), b"0"]+ )+ assert index.has_work("ingestion-task") is task_owned++ await asyncio.wait_for(queue.dequeue(), timeout=1)++ assert backend.write.await_args_list == [+ call("/queue/Semantic/enqueue", queued_payload.encode()),+ call("/queue/Semantic/ack", b"queue-message"),+ ]+ status = await queue.get_status()+ assert status.is_complete+ assert status.in_progress == status.processed == status.requeue_count == 0+ assert status.error_count == 1+ assert [entry.message for entry in status.errors] == [str(error)]+ assert not index.has_work("ingestion-task") # Includes active consumer registration.+ assert index.failure("ingestion-task") == (str(error) if task_owned else None)+ if task_owned:+ finalize.assert_awaited_once()+ assert finalize.call_args.args[0].task_id == "ingestion-task"+ else:+ finalize.assert_not_awaited()+ env.retry_queue.enqueue.assert_not_awaited()+ env.pathlock.pathlock_release.assert_awaited_once_with(env.lease)+ await _assert_request_failed(env, msg, error)+++async def test_malformed_resources_do_not_pause_following_valid_work(semantic_env):+ env = semantic_env+ for wrapped in [False, True, False, True]:+ msg = env.message()+ error = _storage_error(wrapped)+ env.dag.run.side_effect = error+ await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ await _assert_request_failed(env, msg, error)++ env.dag.run.side_effect = None+ valid = env.message()+ await asyncio.wait_for(env.processor.on_dequeue(valid.to_dict()), timeout=1)+ await asyncio.wait_for(env.tracker.wait_for_request(valid.telemetry_id), timeout=1)++ assert env.error.call_count == 4+ env.success.assert_called_once_with()+ env.requeue.assert_not_called()+ env.retry_queue.enqueue.assert_not_awaited()+ assert env.dag.run.await_count == 5+ assert env.tracker.build_queue_status(valid.telemetry_id)["Semantic"]["processed"] == 1+ env.processor._circuit_breaker.check()+++ ("error", "trips_breaker"),+ [(TimeoutError("API timeout"), True), (LockAcquisitionError("tree is locked"), False)],+)+async def test_retryable_errors_keep_existing_behavior(+ semantic_env, monkeypatch, error, trips_breaker+):+ env = semantic_env+ env.processor._circuit_breaker = CircuitBreaker(failure_threshold=1)+ sleep = AsyncMock()+ monkeypatch.setattr("openviking.storage.queuefs.semantic_processor.asyncio.sleep", sleep)+ msg = env.message()+ env.dag.run.side_effect = error++ await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)++ env.retry_queue.enqueue.assert_awaited_once()+ assert env.retry_queue.enqueue.call_args.args[0].id == msg.id+ env.requeue.assert_called_once_with()+ env.success.assert_called_once_with()+ env.error.assert_not_called()+ assert not env.tracker.is_complete(msg.telemetry_id)+ assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"]["requeue_count"] == 1+ if trips_breaker:+ with pytest.raises(CircuitBreakerOpen):+ env.processor._circuit_breaker.check()+ sleep.assert_awaited_once()+ else:+ env.processor._circuit_breaker.check()+ sleep.assert_not_awaited()+++async def test_permanent_api_error_still_opens_breaker(semantic_env):+ env = semantic_env+ msg = env.message()+ error = RuntimeError("400 invalid model parameter")+ env.dag.run.side_effect = error++ await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)++ env.error.assert_called_once()+ env.success.assert_not_called()+ env.retry_queue.enqueue.assert_not_awaited()+ await _assert_request_failed(env, msg, error)+ with pytest.raises(CircuitBreakerOpen):+ env.processor._circuit_breaker.check()+++async def test_open_breaker_requeues_then_recovers(semantic_env, monkeypatch):+ env = semantic_env+ now = [100.0]+ monkeypatch.setattr("openviking.utils.circuit_breaker.time.monotonic", lambda: now[0])+ sleep = AsyncMock()+ monkeypatch.setattr("openviking.storage.queuefs.semantic_processor.asyncio.sleep", sleep)+ breaker = env.processor._circuit_breaker+ breaker.record_failure(RuntimeError("400 invalid model parameter"))+ msg = env.message()++ await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)++ env.dag.run.assert_not_awaited()+ env.retry_queue.enqueue.assert_awaited_once()+ sleep.assert_awaited_once_with(30)+ env.requeue.assert_called_once_with()+ assert not env.tracker.is_complete(msg.telemetry_id)+ with pytest.raises(CircuitBreakerOpen):+ breaker.check()++ now[0] += 301+ await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)++ env.dag.run.assert_awaited_once_with(msg.uri)+ assert env.success.call_count == 2+ env.error.assert_not_called()+ assert breaker.retry_after == 0+ breaker.check()diff --git a/tests/unit/test_model_retry.py b/tests/unit/test_model_retry.pyindex 95b41bb5b..a0c915e07 100644--- a/tests/unit/test_model_retry.py+++ b/tests/unit/test_model_retry.py@@ -4,6 +4,12 @@import pytest+from openviking.pyagfs.exceptions import (+ AGFSClientError,+ AGFSConnectionError,+ AGFSNotADirectoryError,+ AGFSTimeoutError,+)from openviking.utils.exceptions import AllCredentialsFailedErrorfrom openviking.utils.model_retry import (ERROR_CLASS_AUTH,@@ -12,6 +18,7 @@ from openviking.utils.model_retry import (ERROR_CLASS_PERMANENT,ERROR_CLASS_QUOTA_EXCEEDED,ERROR_CLASS_TRANSIENT,+ ERROR_CLASS_UNKNOWN,classify_api_error,extract_metric_error_code,retry_async,@@ -19,6 +26,44 @@ from openviking.utils.model_retry import ()[email protected]("wrapped", [False, True])+ "error_type",+ [+ FileNotFoundError,+ PermissionError,+ IsADirectoryError,+ NotADirectoryError,+ AGFSNotADirectoryError,+ ],+)[email protected]("message", ["/resources/notes.txt", "timeout at /arbitrary/input.md"])+def test_classify_permanent_filesystem_errors(error_type, message, wrapped):+ error = error_type(message)+ if wrapped:+ wrapper = RuntimeError("storage operation failed")+ wrapper.__cause__ = error+ error = wrapper+ assert classify_api_error(error) == ERROR_CLASS_PERMANENT++[email protected]("wrapped", [False, True])+ ("error", "expected"),+ [+ (AGFSTimeoutError("request timeout"), ERROR_CLASS_TRANSIENT),+ (AGFSConnectionError("connection refused"), ERROR_CLASS_TRANSIENT),+ (AGFSClientError("unexpected storage failure"), ERROR_CLASS_UNKNOWN),+ ],+)+def test_classify_other_agfs_errors(error, expected, wrapped):+ if wrapped:+ wrapper = RuntimeError("storage operation failed")+ wrapper.__cause__ = error+ error = wrapper+ assert classify_api_error(error) == expected++def test_classify_api_error_recognizes_request_burst_too_fast():assert classify_api_error(RuntimeError("RequestBurstTooFast")) == ERROR_CLASS_TRANSIENT@@ -33,10 +78,13 @@ class _ProviderError(RuntimeError):def test_extract_metric_error_code_prefers_structured_provider_code():- assert extract_metric_error_code(_ProviderError(status_code=429, code="RateLimitExceeded")) == "429"- assert extract_metric_error_code(_ProviderError(body={"error": {"code": "InvalidParameter"}})) == (- "InvalidParameter"+ assert (+ extract_metric_error_code(_ProviderError(status_code=429, code="RateLimitExceeded"))+ == "429")+ assert extract_metric_error_code(+ _ProviderError(body={"error": {"code": "InvalidParameter"}})+ ) == ("InvalidParameter")def test_extract_metric_error_code_uses_safe_fallbacks_only():