Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: terminate semantic work on AGFS not-a-directory errors (#4934)

Matt Van Hornb0c5e8571113
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index c43fb9036..7fb73d552 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -33,6 +33,7 @@ from openviking.parse.parsers.media.utils import (
get_media_type,
)
from openviking.prompts import render_prompt
+from openviking.pyagfs.exceptions import AGFSNotADirectoryError
from openviking.server.identity import RequestContext, Role
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.service.task_work_index import detach_task_context
@@ -629,10 +630,12 @@ class SemanticProcessor(DequeueHandlerBase):
return ProcessResult.failed(str(e))
elif error_class == ERROR_CLASS_PERMANENT:
logger.critical(
- f"Permanent API error processing semantic message, dropping: {e}",
+ f"Permanent error processing semantic message, dropping: {e}",
exc_info=True,
)
- self._circuit_breaker.record_failure(e)
+ # A malformed filesystem target does not indicate an API outage.
+ if not any(isinstance(exc, AGFSNotADirectoryError) for exc in (e, e.__cause__)):
+ self._circuit_breaker.record_failure(e)
if msg is not None:
self._merge_request_stats(msg.telemetry_id, error_count=1)
get_request_wait_tracker().mark_semantic_failed(
diff --git a/openviking/utils/model_retry.py b/openviking/utils/model_retry.py
index 7f12fbc9b..9b6d7f6e2 100644
--- a/openviking/utils/model_retry.py
+++ b/openviking/utils/model_retry.py
@@ -8,6 +8,7 @@ import threading
import time
from typing import Awaitable, Callable, TypeVar
+from openviking.pyagfs.exceptions import AGFSNotADirectoryError
from openviking.utils.exceptions import AllCredentialsFailedError
logger = logging.getLogger(__name__)
@@ -74,6 +75,7 @@ def extract_metric_error_code(error: BaseException) -> str:
return "connection_error"
return "unknown"
+
INPUT_TOO_LARGE_PATTERNS = (
"413",
"payload too large",
@@ -130,7 +132,13 @@ QUOTA_EXCEEDED_PATTERNS = (
"usage quota",
)
-_PERMANENT_IO_ERRORS = (FileNotFoundError, PermissionError, IsADirectoryError, NotADirectoryError)
+_PERMANENT_IO_ERRORS = (
+ FileNotFoundError,
+ PermissionError,
+ IsADirectoryError,
+ NotADirectoryError,
+ AGFSNotADirectoryError,
+)
TRANSIENT_API_ERROR_PATTERNS = (
"429",
diff --git a/tests/storage/test_semantic_processor_permanent_storage_error.py b/tests/storage/test_semantic_processor_permanent_storage_error.py
new file mode 100644
index 000000000..e78f603c2
--- /dev/null
+++ b/tests/storage/test_semantic_processor_permanent_storage_error.py
@@ -0,0 +1,290 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Permanent storage failures must finish semantic work without blocking the API."""
+
+import asyncio
+import json
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock, call
+from uuid import uuid4
+
+import pytest
+
+from openviking.pyagfs.exceptions import AGFSNotADirectoryError
+from openviking.service.task_work_index import TaskWorkIndex
+from openviking.storage.errors import LockAcquisitionError
+from openviking.storage.queuefs.named_queue import NamedQueue
+from openviking.storage.queuefs.semantic_dag import DagStats
+from openviking.storage.queuefs.semantic_msg import SemanticMsg
+from openviking.storage.queuefs.semantic_processor import SemanticProcessor
+from openviking.storage.viking_fs import SyncDiff
+from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
+from openviking.utils.circuit_breaker import CircuitBreaker, CircuitBreakerOpen
+
+
+def semantic_env(monkeypatch):
+ lease = {"lease_ref": "semantic-lease", "owner_id": "consumer", "owned": True}
+ pathlock = SimpleNamespace(
+ pathlock_adopt=AsyncMock(return_value=lease),
+ pathlock_as_borrowed=AsyncMock(return_value={**lease, "owned": False}),
+ pathlock_release=AsyncMock(),
+ )
+ fs = SimpleNamespace(_async_agfs=pathlock, exists=AsyncMock(return_value=True))
+ monkeypatch.setattr("openviking.storage.queuefs.semantic_processor.get_viking_fs", lambda: fs)
+ monkeypatch.setattr("openviking.storage.queuefs.semantic_lock.get_viking_fs", lambda: fs)
+ dag = SimpleNamespace(run=AsyncMock(), get_stats=Mock(return_value=DagStats()), stale=False)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.SemanticDagExecutor", Mock(return_value=dag)
+ )
+ retry_queue = SimpleNamespace(enqueue=AsyncMock())
+ manager = SimpleNamespace(SEMANTIC="Semantic", get_queue=lambda name: retry_queue)
+ monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: manager)
+ processor = SemanticProcessor()
+ processor._sync_topdown_recursive = AsyncMock(return_value=SyncDiff())
+ success, requeue, error = Mock(), Mock(), Mock()
+ processor.set_callbacks(success, requeue, error)
+ tracker = get_request_wait_tracker()
+ telemetry_ids = []
+
+ def message(**kwargs):
+ msg = SemanticMsg(
+ uri="viking://resources/notes.txt",
+ context_type="resource",
+ telemetry_id=str(uuid4()),
+ propagate_to_parent=False,
+ **kwargs,
+ )
+ telemetry_ids.append(msg.telemetry_id)
+ tracker.register_request(msg.telemetry_id)
+ tracker.register_semantic_root(msg.telemetry_id, msg.id)
+ return msg
+
+ yield SimpleNamespace(
+ processor=processor,
+ pathlock=pathlock,
+ lease=lease,
+ dag=dag,
+ retry_queue=retry_queue,
+ success=success,
+ requeue=requeue,
+ error=error,
+ tracker=tracker,
+ message=message,
+ )
+ for telemetry_id in telemetry_ids:
+ tracker.cleanup(telemetry_id)
+
+
+def _storage_error(wrapped):
+ error = AGFSNotADirectoryError("timeout at /arbitrary/notes.txt")
+ if wrapped:
+ wrapper = RuntimeError("semantic storage operation failed")
+ wrapper.__cause__ = error
+ return wrapper
+ return error
+
+
+async def _assert_request_failed(env, msg, error):
+ await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)
+ assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"] == {
+ "processed": 0,
+ "requeue_count": 0,
+ "error_count": 1,
+ "errors": [{"message": str(error)}],
+ }
+
+
[email protected]("wrapped", [False, True])
[email protected]("boundary", ["sync", "dag"])
[email protected]("owned", [False, True])
+async def test_storage_error_finishes_request_and_respects_lock_ownership(
+ semantic_env, wrapped, boundary, owned
+):
+ env = semantic_env
+ msg = env.message(
+ target_uri="viking://resources/existing.txt" if boundary == "sync" else "",
+ lock_handoff={"owner_id": "producer"} if owned else None,
+ )
+ error = _storage_error(wrapped)
+ operation = env.processor._sync_topdown_recursive if boundary == "sync" else env.dag.run
+ operation.side_effect = error
+ data = msg.to_dict()
+
+ await asyncio.wait_for(
+ env.processor.on_dequeue(data, lock=None if owned else env.lease), timeout=1
+ )
+
+ operation.assert_awaited_once()
+ env.error.assert_called_once_with(str(error), data)
+ env.success.assert_not_called()
+ env.requeue.assert_not_called()
+ env.retry_queue.enqueue.assert_not_awaited()
+ await _assert_request_failed(env, msg, error)
+ env.processor._circuit_breaker.check()
+ if owned:
+ env.pathlock.pathlock_adopt.assert_awaited_once_with(msg.lock_handoff)
+ env.pathlock.pathlock_release.assert_awaited_once_with(env.lease)
+ else:
+ env.pathlock.pathlock_as_borrowed.assert_awaited_once_with(env.lease)
+ env.pathlock.pathlock_release.assert_not_awaited()
+
+
[email protected]("wrapped", [False, True])
[email protected]("task_owned", [False, True])
+async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrapped, task_owned):
+ env = semantic_env
+ msg = env.message(lock_handoff={"owner_id": "producer"})
+ error = _storage_error(wrapped)
+ env.dag.run.side_effect = error
+ backend = SimpleNamespace(mkdir=AsyncMock(), write=AsyncMock(return_value="queue-message"))
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.named_queue.AsyncAGFSClient", lambda agfs: backend
+ )
+ index = TaskWorkIndex()
+ finalize = AsyncMock()
+ index.set_callbacks(
+ finalize_before_ack=finalize, is_cancellation_requested=lambda task_id: False
+ )
+ queue = NamedQueue(
+ object(), "/queue", "Semantic", dequeue_handler=env.processor, task_work_index=index
+ )
+ payload = msg.to_dict()
+ if task_owned:
+ payload["task_id"] = "ingestion-task"
+ await queue.enqueue(payload)
+ queued_payload = backend.write.call_args.args[1].decode()
+ backend.read = AsyncMock(
+ side_effect=[json.dumps({"id": "queue-message", "data": queued_payload}).encode(), b"0"]
+ )
+ assert index.has_work("ingestion-task") is task_owned
+
+ await asyncio.wait_for(queue.dequeue(), timeout=1)
+
+ assert backend.write.await_args_list == [
+ call("/queue/Semantic/enqueue", queued_payload.encode()),
+ call("/queue/Semantic/ack", b"queue-message"),
+ ]
+ status = await queue.get_status()
+ assert status.is_complete
+ assert status.in_progress == status.processed == status.requeue_count == 0
+ assert status.error_count == 1
+ assert [entry.message for entry in status.errors] == [str(error)]
+ assert not index.has_work("ingestion-task") # Includes active consumer registration.
+ assert index.failure("ingestion-task") == (str(error) if task_owned else None)
+ if task_owned:
+ finalize.assert_awaited_once()
+ assert finalize.call_args.args[0].task_id == "ingestion-task"
+ else:
+ finalize.assert_not_awaited()
+ env.retry_queue.enqueue.assert_not_awaited()
+ env.pathlock.pathlock_release.assert_awaited_once_with(env.lease)
+ await _assert_request_failed(env, msg, error)
+
+
+async def test_malformed_resources_do_not_pause_following_valid_work(semantic_env):
+ env = semantic_env
+ for wrapped in [False, True, False, True]:
+ msg = env.message()
+ error = _storage_error(wrapped)
+ env.dag.run.side_effect = error
+ await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)
+ await _assert_request_failed(env, msg, error)
+
+ env.dag.run.side_effect = None
+ valid = env.message()
+ await asyncio.wait_for(env.processor.on_dequeue(valid.to_dict()), timeout=1)
+ await asyncio.wait_for(env.tracker.wait_for_request(valid.telemetry_id), timeout=1)
+
+ assert env.error.call_count == 4
+ env.success.assert_called_once_with()
+ env.requeue.assert_not_called()
+ env.retry_queue.enqueue.assert_not_awaited()
+ assert env.dag.run.await_count == 5
+ assert env.tracker.build_queue_status(valid.telemetry_id)["Semantic"]["processed"] == 1
+ env.processor._circuit_breaker.check()
+
+
+ ("error", "trips_breaker"),
+ [(TimeoutError("API timeout"), True), (LockAcquisitionError("tree is locked"), False)],
+)
+async def test_retryable_errors_keep_existing_behavior(
+ semantic_env, monkeypatch, error, trips_breaker
+):
+ env = semantic_env
+ env.processor._circuit_breaker = CircuitBreaker(failure_threshold=1)
+ sleep = AsyncMock()
+ monkeypatch.setattr("openviking.storage.queuefs.semantic_processor.asyncio.sleep", sleep)
+ msg = env.message()
+ env.dag.run.side_effect = error
+
+ await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)
+
+ env.retry_queue.enqueue.assert_awaited_once()
+ assert env.retry_queue.enqueue.call_args.args[0].id == msg.id
+ env.requeue.assert_called_once_with()
+ env.success.assert_called_once_with()
+ env.error.assert_not_called()
+ assert not env.tracker.is_complete(msg.telemetry_id)
+ assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"]["requeue_count"] == 1
+ if trips_breaker:
+ with pytest.raises(CircuitBreakerOpen):
+ env.processor._circuit_breaker.check()
+ sleep.assert_awaited_once()
+ else:
+ env.processor._circuit_breaker.check()
+ sleep.assert_not_awaited()
+
+
+async def test_permanent_api_error_still_opens_breaker(semantic_env):
+ env = semantic_env
+ msg = env.message()
+ error = RuntimeError("400 invalid model parameter")
+ env.dag.run.side_effect = error
+
+ await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)
+
+ env.error.assert_called_once()
+ env.success.assert_not_called()
+ env.retry_queue.enqueue.assert_not_awaited()
+ await _assert_request_failed(env, msg, error)
+ with pytest.raises(CircuitBreakerOpen):
+ env.processor._circuit_breaker.check()
+
+
+async def test_open_breaker_requeues_then_recovers(semantic_env, monkeypatch):
+ env = semantic_env
+ now = [100.0]
+ monkeypatch.setattr("openviking.utils.circuit_breaker.time.monotonic", lambda: now[0])
+ sleep = AsyncMock()
+ monkeypatch.setattr("openviking.storage.queuefs.semantic_processor.asyncio.sleep", sleep)
+ breaker = env.processor._circuit_breaker
+ breaker.record_failure(RuntimeError("400 invalid model parameter"))
+ msg = env.message()
+
+ await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)
+
+ env.dag.run.assert_not_awaited()
+ env.retry_queue.enqueue.assert_awaited_once()
+ sleep.assert_awaited_once_with(30)
+ env.requeue.assert_called_once_with()
+ assert not env.tracker.is_complete(msg.telemetry_id)
+ with pytest.raises(CircuitBreakerOpen):
+ breaker.check()
+
+ now[0] += 301
+ await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)
+ await asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)
+
+ env.dag.run.assert_awaited_once_with(msg.uri)
+ assert env.success.call_count == 2
+ env.error.assert_not_called()
+ assert breaker.retry_after == 0
+ breaker.check()
diff --git a/tests/unit/test_model_retry.py b/tests/unit/test_model_retry.py
index 95b41bb5b..a0c915e07 100644
--- a/tests/unit/test_model_retry.py
+++ b/tests/unit/test_model_retry.py
@@ -4,6 +4,12 @@
import pytest
+from openviking.pyagfs.exceptions import (
+ AGFSClientError,
+ AGFSConnectionError,
+ AGFSNotADirectoryError,
+ AGFSTimeoutError,
+)
from openviking.utils.exceptions import AllCredentialsFailedError
from openviking.utils.model_retry import (
ERROR_CLASS_AUTH,
@@ -12,6 +18,7 @@ from openviking.utils.model_retry import (
ERROR_CLASS_PERMANENT,
ERROR_CLASS_QUOTA_EXCEEDED,
ERROR_CLASS_TRANSIENT,
+ ERROR_CLASS_UNKNOWN,
classify_api_error,
extract_metric_error_code,
retry_async,
@@ -19,6 +26,44 @@ from openviking.utils.model_retry import (
)
[email protected]("wrapped", [False, True])
+ "error_type",
+ [
+ FileNotFoundError,
+ PermissionError,
+ IsADirectoryError,
+ NotADirectoryError,
+ AGFSNotADirectoryError,
+ ],
+)
[email protected]("message", ["/resources/notes.txt", "timeout at /arbitrary/input.md"])
+def test_classify_permanent_filesystem_errors(error_type, message, wrapped):
+ error = error_type(message)
+ if wrapped:
+ wrapper = RuntimeError("storage operation failed")
+ wrapper.__cause__ = error
+ error = wrapper
+ assert classify_api_error(error) == ERROR_CLASS_PERMANENT
+
+
[email protected]("wrapped", [False, True])
+ ("error", "expected"),
+ [
+ (AGFSTimeoutError("request timeout"), ERROR_CLASS_TRANSIENT),
+ (AGFSConnectionError("connection refused"), ERROR_CLASS_TRANSIENT),
+ (AGFSClientError("unexpected storage failure"), ERROR_CLASS_UNKNOWN),
+ ],
+)
+def test_classify_other_agfs_errors(error, expected, wrapped):
+ if wrapped:
+ wrapper = RuntimeError("storage operation failed")
+ wrapper.__cause__ = error
+ error = wrapper
+ assert classify_api_error(error) == expected
+
+
def test_classify_api_error_recognizes_request_burst_too_fast():
assert classify_api_error(RuntimeError("RequestBurstTooFast")) == ERROR_CLASS_TRANSIENT
@@ -33,10 +78,13 @@ class _ProviderError(RuntimeError):
def test_extract_metric_error_code_prefers_structured_provider_code():
- assert extract_metric_error_code(_ProviderError(status_code=429, code="RateLimitExceeded")) == "429"
- assert extract_metric_error_code(_ProviderError(body={"error": {"code": "InvalidParameter"}})) == (
- "InvalidParameter"
+ assert (
+ extract_metric_error_code(_ProviderError(status_code=429, code="RateLimitExceeded"))
+ == "429"
)
+ assert extract_metric_error_code(
+ _ProviderError(body={"error": {"code": "InvalidParameter"}})
+ ) == ("InvalidParameter")
def test_extract_metric_error_code_uses_safe_fallbacks_only():