Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
test(queuefs): align semantic task tests with current APIs (#5410)
Wu Shuwenf708ecfffa02
diff --git a/tests/service/test_task_processing_time.py b/tests/service/test_task_processing_time.pyindex 31d32531d..35956ee10 100644--- a/tests/service/test_task_processing_time.py+++ b/tests/service/test_task_processing_time.py@@ -10,7 +10,10 @@ import pytestfrom openviking.service.task_processing_time import ProcessingClockfrom openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.service.task_work_index import TaskWorkIndex, bind_task_context-from openviking.storage.queuefs.semantic_dag import SemanticDagExecutor, SemanticNodeScheduler+from openviking.storage.queuefs.semantic_executor import (+ SemanticTreeExecutor,+ SemanticTreeScheduler,+)from openviking.telemetry.request_wait_tracker import RequestWaitTracker@@ -46,12 +49,12 @@ async def test_saturated_dag_scheduler_counts_only_dispatched_nodes(monkeypatch)SimpleNamespace(monotonic=lambda: now[0]),)index = TaskWorkIndex()- scheduler = SemanticNodeScheduler(max_workers=1)+ scheduler = SemanticTreeScheduler(max_workers=1)monkeypatch.setattr(- "openviking.storage.queuefs.semantic_dag.get_semantic_node_scheduler",+ "openviking.storage.queuefs.semantic_executor.get_semantic_tree_scheduler",lambda _: scheduler,)- monkeypatch.setattr("openviking.storage.queuefs.semantic_dag.get_viking_fs", lambda: None)+ monkeypatch.setattr("openviking.storage.queuefs.semantic_executor.get_viking_fs", lambda: None)entered = {name: asyncio.Event() for name in ["a", "b"]}release = {name: asyncio.Event() for name in ["a", "b"]}queued = asyncio.Event()@@ -59,7 +62,7 @@ async def test_saturated_dag_scheduler_counts_only_dispatched_nodes(monkeypatch)for name in ["a", "b"]:index.init_processing(name)with bind_task_context(name, "account", "user"):- executor = SemanticDagExecutor(None, "resource", 1, None)+ executor = SemanticTreeExecutor(None, "resource", 1, None)executors[name] = executorasync def node(_work, name=name, executor=executor):@@ -165,7 +168,9 @@ async def test_nested_wait_and_cancellation_restore_processing_owner(monkeypatch@pytest.mark.asyncio@pytest.mark.parametrize("custom_enqueue", [False, True])-async def test_semantic_retry_cooldown_excludes_wait_but_counts_enqueue(monkeypatch, custom_enqueue):+async def test_semantic_retry_cooldown_excludes_wait_but_counts_enqueue(+ monkeypatch, custom_enqueue+):from openviking.storage.queuefs.semantic_processor import SemanticProcessornow = [0.0]@@ -175,7 +180,7 @@ async def test_semantic_retry_cooldown_excludes_wait_but_counts_enqueue(monkeypa)index = TaskWorkIndex()index.init_processing("task")- message = SimpleNamespace(uri="viking://resources/retry-test")+ message = SimpleNamespace(uri="viking://resources/retry-test", account_id="account")enqueued = []async def sleep(delay):@@ -199,7 +204,9 @@ async def test_semantic_retry_cooldown_excludes_wait_but_counts_enqueue(monkeypa"openviking.storage.queuefs.get_queue_manager",lambda: SimpleNamespace(SEMANTIC="semantic", get_queue=lambda _: queue),)- processor = SimpleNamespace(_circuit_breaker=SimpleNamespace(retry_after=30))+ processor = SimpleNamespace(+ _account_breaker=lambda _account_id: SimpleNamespace(retry_after=30)+ )worker = asyncio.current_task()index.register_active("task", worker)try:diff --git a/tests/storage/test_semantic_processor_permanent_storage_error.py b/tests/storage/test_semantic_processor_permanent_storage_error.pyindex e78f603c2..971400f6c 100644--- a/tests/storage/test_semantic_processor_permanent_storage_error.py+++ b/tests/storage/test_semantic_processor_permanent_storage_error.py@@ -11,10 +11,12 @@ from uuid import uuid4import pytestfrom openviking.pyagfs.exceptions import AGFSNotADirectoryError+from openviking.service.task_queue_middleware import TaskWorkQueueMiddlewarefrom openviking.service.task_work_index import TaskWorkIndexfrom openviking.storage.errors import LockAcquisitionErrorfrom openviking.storage.queuefs.named_queue import NamedQueue-from openviking.storage.queuefs.semantic_dag import DagStats+from openviking.storage.queuefs.process_result import ProcessOutcome+from openviking.storage.queuefs.semantic_executor import SemanticTreeStatsfrom openviking.storage.queuefs.semantic_msg import SemanticMsgfrom openviking.storage.queuefs.semantic_processor import SemanticProcessorfrom openviking.storage.viking_fs import SyncDiff@@ -33,17 +35,19 @@ def semantic_env(monkeypatch):fs = SimpleNamespace(_async_agfs=pathlock, exists=AsyncMock(return_value=True))monkeypatch.setattr("openviking.storage.queuefs.semantic_processor.get_viking_fs", lambda: fs)monkeypatch.setattr("openviking.storage.queuefs.semantic_lock.get_viking_fs", lambda: fs)- dag = SimpleNamespace(run=AsyncMock(), get_stats=Mock(return_value=DagStats()), stale=False)+ dag = SimpleNamespace(+ run=AsyncMock(), get_stats=Mock(return_value=SemanticTreeStats()), stale=False+ )monkeypatch.setattr(- "openviking.storage.queuefs.semantic_processor.SemanticDagExecutor", Mock(return_value=dag)+ "openviking.storage.queuefs.semantic_processor.SemanticTreeExecutor",+ Mock(return_value=dag),)retry_queue = SimpleNamespace(enqueue=AsyncMock())manager = SimpleNamespace(SEMANTIC="Semantic", get_queue=lambda name: retry_queue)monkeypatch.setattr("openviking.storage.queuefs.get_queue_manager", lambda: manager)- processor = SemanticProcessor()+ resolver = SimpleNamespace(get_vlm=AsyncMock(return_value=SimpleNamespace()))+ processor = SemanticProcessor(vlm_resolver=resolver)processor._sync_topdown_recursive = AsyncMock(return_value=SyncDiff())- success, requeue, error = Mock(), Mock(), Mock()- processor.set_callbacks(success, requeue, error)tracker = get_request_wait_tracker()telemetry_ids = []@@ -66,9 +70,6 @@ def semantic_env(monkeypatch):lease=lease,dag=dag,retry_queue=retry_queue,- success=success,- requeue=requeue,- error=error,tracker=tracker,message=message,)@@ -112,17 +113,16 @@ async def test_storage_error_finishes_request_and_respects_lock_ownership(operation.side_effect = errordata = msg.to_dict()- await asyncio.wait_for(+ result = await asyncio.wait_for(env.processor.on_dequeue(data, lock=None if owned else env.lease), timeout=1)+ assert result.outcome is ProcessOutcome.FAILED+ assert result.error == str(error)operation.assert_awaited_once()- env.error.assert_called_once_with(str(error), data)- env.success.assert_not_called()- env.requeue.assert_not_called()env.retry_queue.enqueue.assert_not_awaited()await _assert_request_failed(env, msg, error)- env.processor._circuit_breaker.check()+ env.processor._account_breaker(msg.account_id).check()if owned:env.pathlock.pathlock_adopt.assert_awaited_once_with(msg.lock_handoff)env.pathlock.pathlock_release.assert_awaited_once_with(env.lease)@@ -149,7 +149,11 @@ async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrappefinalize_before_ack=finalize, is_cancellation_requested=lambda task_id: False)queue = NamedQueue(- object(), "/queue", "Semantic", dequeue_handler=env.processor, task_work_index=index+ object(),+ "/queue",+ "Semantic",+ dequeue_handler=env.processor,+ middlewares=[TaskWorkQueueMiddleware(index)],)payload = msg.to_dict()if task_owned:@@ -157,7 +161,10 @@ async def test_queue_acks_failed_semantic_work(semantic_env, monkeypatch, wrappeawait queue.enqueue(payload)queued_payload = backend.write.call_args.args[1].decode()backend.read = AsyncMock(- side_effect=[json.dumps({"id": "queue-message", "data": queued_payload}).encode(), b"0"]+ side_effect=[+ json.dumps({"id": "queue-message", "data": queued_payload}).encode(),+ json.dumps({"pending": 0, "processing": 0}).encode(),+ ])assert index.has_work("ingestion-task") is task_owned@@ -191,21 +198,21 @@ async def test_malformed_resources_do_not_pause_following_valid_work(semantic_enmsg = env.message()error = _storage_error(wrapped)env.dag.run.side_effect = error- await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ assert result.outcome is ProcessOutcome.FAILED+ assert result.error == str(error)await _assert_request_failed(env, msg, error)env.dag.run.side_effect = Nonevalid = env.message()- await asyncio.wait_for(env.processor.on_dequeue(valid.to_dict()), timeout=1)+ result = await asyncio.wait_for(env.processor.on_dequeue(valid.to_dict()), timeout=1)+ assert result.outcome is ProcessOutcome.SUCCESSawait asyncio.wait_for(env.tracker.wait_for_request(valid.telemetry_id), timeout=1)- assert env.error.call_count == 4- env.success.assert_called_once_with()- env.requeue.assert_not_called()env.retry_queue.enqueue.assert_not_awaited()assert env.dag.run.await_count == 5assert env.tracker.build_queue_status(valid.telemetry_id)["Semantic"]["processed"] == 1- env.processor._circuit_breaker.check()+ env.processor._account_breaker(valid.account_id).check()@pytest.mark.asyncio@@ -217,27 +224,26 @@ async def test_retryable_errors_keep_existing_behavior(semantic_env, monkeypatch, error, trips_breaker):env = semantic_env- env.processor._circuit_breaker = CircuitBreaker(failure_threshold=1)sleep = AsyncMock()monkeypatch.setattr("openviking.storage.queuefs.semantic_processor.asyncio.sleep", sleep)msg = env.message()+ breaker = CircuitBreaker(failure_threshold=1)+ env.processor._circuit_breakers[msg.account_id] = breakerenv.dag.run.side_effect = error- await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ assert result.outcome is ProcessOutcome.REQUEUEDenv.retry_queue.enqueue.assert_awaited_once()assert env.retry_queue.enqueue.call_args.args[0].id == msg.id- env.requeue.assert_called_once_with()- env.success.assert_called_once_with()- env.error.assert_not_called()assert not env.tracker.is_complete(msg.telemetry_id)assert env.tracker.build_queue_status(msg.telemetry_id)["Semantic"]["requeue_count"] == 1if trips_breaker:with pytest.raises(CircuitBreakerOpen):- env.processor._circuit_breaker.check()+ breaker.check()sleep.assert_awaited_once()else:- env.processor._circuit_breaker.check()+ breaker.check()sleep.assert_not_awaited()@@ -248,14 +254,14 @@ async def test_permanent_api_error_still_opens_breaker(semantic_env):error = RuntimeError("400 invalid model parameter")env.dag.run.side_effect = error- await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)- env.error.assert_called_once()- env.success.assert_not_called()+ assert result.outcome is ProcessOutcome.FAILED+ assert result.error == str(error)env.retry_queue.enqueue.assert_not_awaited()await _assert_request_failed(env, msg, error)with pytest.raises(CircuitBreakerOpen):- env.processor._circuit_breaker.check()+ env.processor._account_breaker(msg.account_id).check()@pytest.mark.asyncio@@ -265,26 +271,25 @@ async def test_open_breaker_requeues_then_recovers(semantic_env, monkeypatch):monkeypatch.setattr("openviking.utils.circuit_breaker.time.monotonic", lambda: now[0])sleep = AsyncMock()monkeypatch.setattr("openviking.storage.queuefs.semantic_processor.asyncio.sleep", sleep)- breaker = env.processor._circuit_breaker- breaker.record_failure(RuntimeError("400 invalid model parameter"))msg = env.message()+ breaker = env.processor._account_breaker(msg.account_id)+ breaker.record_failure(RuntimeError("400 invalid model parameter"))- await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ assert result.outcome is ProcessOutcome.REQUEUEDenv.dag.run.assert_not_awaited()env.retry_queue.enqueue.assert_awaited_once()sleep.assert_awaited_once_with(30)- env.requeue.assert_called_once_with()assert not env.tracker.is_complete(msg.telemetry_id)with pytest.raises(CircuitBreakerOpen):breaker.check()now[0] += 301- await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ result = await asyncio.wait_for(env.processor.on_dequeue(msg.to_dict()), timeout=1)+ assert result.outcome is ProcessOutcome.SUCCESSawait asyncio.wait_for(env.tracker.wait_for_request(msg.telemetry_id), timeout=1)env.dag.run.assert_awaited_once_with(msg.uri)- assert env.success.call_count == 2- env.error.assert_not_called()assert breaker.retry_after == 0breaker.check()