Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(compile): 同库同目标任务轮转排队 (#4863)
Qin Haojie39670e7b95a2
diff --git a/openviking/service/compile_service.py b/openviking/service/compile_service.pyindex 6cb746c26..860264fa9 100644--- a/openviking/service/compile_service.py+++ b/openviking/service/compile_service.py@@ -257,6 +257,9 @@ class CompileService:def poll_interval_seconds(self) -> float:return self._endpoint().poll_interval_ms / 1000.0+ def serialization_key(self, payload: Mapping[str, Any]) -> str:+ return payload["to"]+def configure_local_backend(self, base_url: str, gateway_token: str) -> None:if self._config.base_url:returndiff --git a/openviking/service/core.py b/openviking/service/core.pyindex e1a1deac3..3d80f4861 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -526,7 +526,8 @@ class OpenVikingService:self._queue_manager.USER_DELETION,allow_create=True,)- await self._queue_manager.prepare_task_tracking(get_task_tracker())+ restored_tasks = await self._queue_manager.prepare_task_tracking(get_task_tracker())+ await self._external_task_service.restore_tasks(restored_tasks)if self._config.enable_watch_scheduler:await self._watch_scheduler.start()diff --git a/openviking/service/external_task_service.py b/openviking/service/external_task_service.pyindex c1689e05e..009b6359d 100644--- a/openviking/service/external_task_service.py+++ b/openviking/service/external_task_service.py@@ -6,7 +6,7 @@ from __future__ import annotationsimport asynciofrom dataclasses import dataclass-from typing import Any, Awaitable, Callable, Mapping, Protocol+from typing import Any, Awaitable, Callable, Iterable, Mapping, Protocolfrom uuid import uuid4from openviking.server.identity import RequestContext@@ -51,6 +51,8 @@ class ExternalTaskProvider(Protocol):@propertydef poll_interval_seconds(self) -> float: ...+ def serialization_key(self, payload: Mapping[str, Any]) -> str | None: ...+async def submit(self,ov_task_id: str,@@ -77,6 +79,10 @@ class ExternalTaskService:def __init__(self) -> None:self._providers: dict[str, ExternalTaskProvider] = {}+ # Accessed only on the service loop. Restored owners may include multiple+ # tasks submitted before target serialization was enabled.+ self._owners: dict[tuple[str | None, str, str], set[str]] = {}+ self._executing: set[str] = set()def register(self, provider: ExternalTaskProvider) -> None:if provider.task_type in self._providers:@@ -93,6 +99,36 @@ class ExternalTaskService:)return provider+ def _serialization_key(self, task: TaskRecord) -> tuple[str | None, str, str] | None:+ key = self._provider(task.task_type).serialization_key(task.meta["request"])+ return (task.account_id, task.task_type, key) if key is not None else None++ async def restore_tasks(self, tasks: Iterable[TaskRecord]) -> None:+ """Reserve submitted work before any recovered queue delivery can run."""+ for task in tasks:+ if (+ task.task_type not in self._providers+ or task.result is not None+ or task.error is not None+ ):+ continue+ if task.status not in {TaskStatus.RUNNING, TaskStatus.CANCELLING}:+ continue+ if task.status == TaskStatus.CANCELLING and task.stage in {None, "queued"}:+ auth = await self._task_auth(task.task_id, task.account_id, task.user_id)+ if not auth.get("external_task_id"):+ continue+ key = self._serialization_key(task)+ if key is not None:+ self._owners.setdefault(key, set()).add(task.task_id)++ def _release(self, task: TaskRecord) -> None:+ key = self._serialization_key(task)+ if key is not None and key in self._owners:+ self._owners[key].discard(task.task_id)+ if not self._owners[key]:+ del self._owners[key]+async def create(self,task_type: str,@@ -119,6 +155,12 @@ class ExternalTaskService:)enqueued = Falsetry:+ await tracker.update_stage(+ task.task_id,+ "queued",+ account_id=ctx.account_id,+ user_id=ctx.user.user_id,+ )await get_queue_manager().enqueue(QueueManager.EXTERNAL_TASK,{@@ -128,12 +170,6 @@ class ExternalTaskService:},)enqueued = True- await tracker.update_stage(- task.task_id,- "queued",- account_id=ctx.account_id,- user_id=ctx.user.user_id,- )current = await tracker.get(task.task_id,account_id=ctx.account_id,@@ -152,31 +188,67 @@ class ExternalTaskService:raisereturn task- async def execute(self, task_id: str, account_id: str, user_id: str) -> None:+ async def execute(self, task_id: str, account_id: str, user_id: str) -> bool:+ """Return False when this delivery must rotate behind other target work."""tracker = get_task_tracker()- task = await tracker.get(task_id, account_id=account_id, user_id=user_id)- if task is None or task.status in {- TaskStatus.COMPLETED,- TaskStatus.FAILED,- TaskStatus.CANCELLED,- }:- return- # The outcome is persisted before QueueFS ACK removes the owned work.- # After a crash in that window, let the recovered delivery ACK without- # resubmitting credentials that have already been cleared.- if task.result is not None or task.error is not None:- return+ claimed = False+ try:+ task = await tracker.get(task_id, account_id=account_id, user_id=user_id)+ if task is None or task.status in {+ TaskStatus.COMPLETED,+ TaskStatus.FAILED,+ TaskStatus.CANCELLED,+ }:+ return True+ # The outcome is persisted before QueueFS ACK removes the owned work.+ # After a crash in that window, let the recovered delivery ACK without+ # resubmitting credentials that have already been cleared.+ if task.result is not None or task.error is not None:+ return True++ provider = self._provider(task.task_type)+ payload = task.meta.get("request")+ if not isinstance(payload, dict):+ await tracker.fail(+ task_id,+ "INVALID_ARGUMENT: External task request is missing",+ account_id=account_id,+ user_id=user_id,+ )+ return True+ key = self._serialization_key(task)+ owners = self._owners.get(key, set()) if key is not None else set()+ if task_id in self._executing or (owners and task_id not in owners):+ return False+ # No await between checking and claiming: concurrent deliveries on the+ # service loop cannot both acquire an unoccupied target.+ if key is not None:+ self._owners.setdefault(key, set()).add(task_id)+ self._executing.add(task_id)+ claimed = True+ await self._execute_task(task, provider, payload, account_id, user_id)+ return True+ except asyncio.CancelledError:+ if not tracker.is_cancellation_requested(task_id):+ raise+ await run_to_completion(lambda: self.cancel_recovered(task_id, account_id, user_id))+ raise+ finally:+ # An interrupted delivery keeps ownership until recovery confirms+ # the Runtime has stopped; it must not admit another target writer.+ if claimed:+ self._executing.remove(task_id)- provider = self._provider(task.task_type)- payload = task.meta.get("request")- if not isinstance(payload, dict):- await tracker.fail(- task_id,- "INVALID_ARGUMENT: External task request is missing",- account_id=account_id,- user_id=user_id,- )- return+ async def _execute_task(+ self,+ task: TaskRecord,+ provider: ExternalTaskProvider,+ payload: Mapping[str, Any],+ account_id: str,+ user_id: str,+ ) -> None:+ tracker = get_task_tracker()+ task_id = task.task_idauth = await self._task_auth(task_id, account_id, user_id)connection = self._mapping(auth.get("openviking_connection"))private_payload = self._mapping(auth.get("external_request_private"))@@ -219,33 +291,24 @@ class ExternalTaskService:returnawait asyncio.sleep(provider.poll_interval_seconds)except ExternalTaskError as exc:- await tracker.fail(- task_id,- self._format_error(exc.code, str(exc)),+ await self._apply_snapshot(+ ExternalTaskSnapshot(status="failed", error_code=exc.code, error_message=str(exc)),+ task_id=task_id,account_id=account_id,user_id=user_id,)- except asyncio.CancelledError:- if not tracker.is_cancellation_requested(task_id):- raise- await run_to_completion(- lambda: self._cancel_external(- provider,- ov_task_id=task_id,- payload=payload,- private_payload=private_payload,- connection=connection,- external_task_id=external_task_id,- account_id=account_id,- user_id=user_id,- )- )- raiseasync def cancel_recovered(self, task_id: str, account_id: str, user_id: str) -> None:tracker = get_task_tracker()task = await tracker.get(task_id, account_id=account_id, user_id=user_id)- if task is None or task.stage in {None, "queued"}:+ if task is None:+ return+ if (+ task.status in {TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED}+ or task.result is not None+ or task.error is not None+ ):+ self._release(task)returnpayload = task.meta.get("request")if not isinstance(payload, dict):@@ -255,6 +318,9 @@ class ExternalTaskService:transient=False,)auth = await self._task_auth(task_id, account_id, user_id)+ if task.stage in {None, "queued"} and not auth.get("external_task_id"):+ self._release(task)+ returnawait self._cancel_external(self._provider(task.task_type),ov_task_id=task_id,@@ -287,12 +353,8 @@ class ExternalTaskService:user_id: str,) -> bool:tracker = get_task_tracker()+ task = await tracker.get(task_id, account_id=account_id, user_id=user_id)if snapshot.stage is not None or snapshot.meta:- task = await tracker.get(- task_id,- account_id=account_id,- user_id=user_id,- )stage = snapshot.stage or snapshot.statusmeta = snapshot.meta or {}if task is not None and (@@ -308,37 +370,42 @@ class ExternalTaskService:)if snapshot.status in _ACTIVE_STATUSES:return False- if snapshot.status == "completed":- await tracker.complete(- task_id,- snapshot.result or snapshot.meta or {},- account_id=account_id,- user_id=user_id,- )- return True- if snapshot.status == "failed":- await tracker.fail(- task_id,- self._format_error(- snapshot.error_code or "UNKNOWN",- snapshot.error_message or "External task failed",- ),- account_id=account_id,- user_id=user_id,- )- return True- if snapshot.status == "cancelled":- await tracker.record_cancelled(- task_id,- account_id=account_id,- user_id=user_id,+ if snapshot.status not in {"completed", "failed", "cancelled"}:+ raise ExternalTaskError(+ "INVALID_RESPONSE",+ f"Unknown external task status: {snapshot.status}",+ transient=False,)- return True- raise ExternalTaskError(- "INVALID_RESPONSE",- f"Unknown external task status: {snapshot.status}",- transient=False,- )++ async def finalize() -> None:+ if snapshot.status == "failed":+ await tracker.fail(+ task_id,+ self._format_error(+ snapshot.error_code or "UNKNOWN",+ snapshot.error_message or "External task failed",+ ),+ account_id=account_id,+ user_id=user_id,+ )+ elif snapshot.status == "completed":+ await tracker.complete(+ task_id,+ snapshot.result or snapshot.meta or {},+ account_id=account_id,+ user_id=user_id,+ )+ else:+ await tracker.record_cancelled(+ task_id,+ account_id=account_id,+ user_id=user_id,+ )+ if task is not None:+ self._release(task)++ await run_to_completion(finalize)+ return Trueasync def _cancel_external(self,diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.pyindex 3ea7911ef..a4b8c88db 100644--- a/openviking/service/task_tracker.py+++ b/openviking/service/task_tracker.py@@ -214,10 +214,14 @@ class TaskTracker:self._dispatcher.bind_current_loop()self._install_work_index_callbacks()- async def restore_work_tasks(self, owners: Dict[str, tuple[str, str]]) -> None:+ async def restore_work_tasks(self, owners: Dict[str, tuple[str, str]]) -> List[TaskRecord]:"""Restore task records referenced by rebuilt QueueFS work."""+ restored = []for task_id, (account_id, user_id) in owners.items():- await self.get(task_id, account_id=account_id, user_id=user_id)+ task = await self.get(task_id, account_id=account_id, user_id=user_id)+ if task is not None:+ restored.append(task)+ return restoredasync def _finalize_before_ack(self, metadata: QueueTaskMetadata) -> None:"""Persist the terminal state before QueueFS removes the last recovery message."""diff --git a/openviking/storage/queuefs/external_task_processor.py b/openviking/storage/queuefs/external_task_processor.pyindex fc4031fea..2480e5f66 100644--- a/openviking/storage/queuefs/external_task_processor.py+++ b/openviking/storage/queuefs/external_task_processor.py@@ -6,10 +6,12 @@ from __future__ import annotationsimport asyncioimport json-from typing import Any, Callable, Coroutine, Dict, Optional+from typing import Any, Dict, Optionalfrom openviking.service.external_task_service import ExternalTaskServicefrom openviking.service.task_tracker_concurrency import OwnerLoopDispatcher+from openviking.service.task_work_index import TaskWorkRejected+from openviking.storage.queuefs import QueueManager, get_queue_managerfrom openviking.storage.queuefs.named_queue import DequeueHandlerBase@@ -38,12 +40,6 @@ class ExternalTaskProcessor(DequeueHandlerBase):raise ValueError("External task queue payload is missing task ownership")return task_id, account_id, user_id- async def _run_on_service_loop(- self,- factory: Callable[[], Coroutine[Any, Any, None]],- ) -> None:- await self._dispatcher.run(factory)-async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:if not data:return None@@ -52,7 +48,22 @@ class ExternalTaskProcessor(DequeueHandlerBase):except (json.JSONDecodeError, TypeError, ValueError) as exc:self.report_error(str(exc), data)return None- await self._run_on_service_loop(lambda: self._service.execute(task_id, account_id, user_id))+ processed = await self._dispatcher.run(+ lambda: self._service.execute(task_id, account_id, user_id)+ )+ if not processed:+ # Register a new delivery before the old one is ACKed. Keeping the+ # task ID preserves cancellation and task-owned work across rotation.+ try:+ await get_queue_manager().enqueue(+ QueueManager.EXTERNAL_TASK,+ {"task_id": task_id, "account_id": account_id, "user_id": user_id},+ )+ except TaskWorkRejected:+ # Cancellation only needs the current delivery to be ACKed.+ pass+ else:+ self.report_requeue()self.report_success()return None@@ -64,7 +75,7 @@ class ExternalTaskProcessor(DequeueHandlerBase):except (json.JSONDecodeError, TypeError, ValueError) as exc:self.report_error(str(exc), data)return None- await self._run_on_service_loop(+ await self._dispatcher.run(lambda: self._service.cancel_recovered(task_id, account_id, user_id))self.report_success()diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.pyindex bf7e3e430..670e1a4b8 100644--- a/openviking/storage/queuefs/queue_manager.py+++ b/openviking/storage/queuefs/queue_manager.py@@ -87,8 +87,8 @@ class QueueManager:SESSION_COMMIT = "SessionCommit"EXTERNAL_TASK = "ExternalTask"USER_DELETION = "UserDeletion"- # A deferred archive re-enqueues itself; throttle the next scheduling round.- _SESSION_COMMIT_POLL_INTERVAL = 1.0+ # Deferred work re-enqueues itself; throttle the next scheduling round.+ _REQUEUE_POLL_INTERVAL = 1.0def __init__(self,@@ -137,12 +137,12 @@ class QueueManager:logger.info(f"[QueueManager] mount_point={self.mount_point} Started")- async def prepare_task_tracking(self, tracker: Any) -> None:+ async def prepare_task_tracking(self, tracker: Any) -> list[Any]:"""Rebuild task work from QueueFS before any consumer starts."""snapshots = {name: await queue.snapshot() for name, queue in self._queues.items()}owners = self._task_work_index.rebuild(snapshots)tracker.attach_work_index(self._task_work_index)- await tracker.restore_work_tasks(owners)+ return await tracker.restore_work_tasks(owners)def setup_standard_queues(self, vector_store: Any, start: bool = True) -> None:"""@@ -225,8 +225,8 @@ class QueueManager:loop = asyncio.new_event_loop()asyncio.set_event_loop(loop)poll_interval = (- self._SESSION_COMMIT_POLL_INTERVAL- if queue.name == self.SESSION_COMMIT+ self._REQUEUE_POLL_INTERVAL+ if queue.name in {self.SESSION_COMMIT, self.EXTERNAL_TASK}else self._poll_interval)try:@@ -242,7 +242,7 @@ class QueueManager:data = loop.run_until_complete(queue.dequeue())if data is not None:logger.debug("[QueueManager] Dequeued message from %s", queue.name)- if queue.name == self.SESSION_COMMIT:+ if queue.name in {self.SESSION_COMMIT, self.EXTERNAL_TASK}:stop_event.wait(poll_interval)else:stop_event.wait(poll_interval)@@ -261,8 +261,8 @@ class QueueManager:A Semaphore caps inflight tasks at max_concurrent."""poll_interval = (- self._SESSION_COMMIT_POLL_INTERVAL- if queue.name == self.SESSION_COMMIT+ self._REQUEUE_POLL_INTERVAL+ if queue.name in {self.SESSION_COMMIT, self.EXTERNAL_TASK}else self._poll_interval)sem = asyncio.Semaphore(max_concurrent)diff --git a/tests/server/test_bot_proxy_auth.py b/tests/server/test_bot_proxy_auth.pyindex 80b3ad9c5..66d87a4f8 100644--- a/tests/server/test_bot_proxy_auth.py+++ b/tests/server/test_bot_proxy_auth.py@@ -3,6 +3,9 @@"""Regression tests for bot proxy endpoint auth enforcement."""+import asyncio+import json+from collections import dequefrom types import SimpleNamespaceimport httpx@@ -15,11 +18,16 @@ import openviking.service.compile_service as compile_service_moduleimport openviking.service.external_task_service as external_task_service_modulefrom openviking.server.auth.plugins import DevAuthPlugin, TrustedAuthPluginfrom openviking.server.config import ServerConfig-from openviking.server.identity import AuthMode+from openviking.server.identity import AuthMode, RequestContext, Rolefrom openviking.service.compile_service import CompileServicefrom openviking.service.external_task_service import ExternalTaskService-from openviking.service.task_tracker import TaskRecord, TaskStatus+from openviking.service.task_store import PersistentTaskStore+from openviking.service.task_tracker import TaskRecord, TaskStatus, TaskTracker+from openviking.storage.queuefs import QueueManager+from openviking.storage.queuefs.external_task_processor import ExternalTaskProcessor+from openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils.config.open_viking_config import CompileApiConfig+from tests.utils.mock_agfs import MockLocalAGFSdef test_set_bot_api_key_updates_module_state():@@ -287,7 +295,10 @@ async def test_compile_route_uses_ov_owned_task_and_rejects_legacy_routes(monkey@pytest.mark.asyncio-async def test_compile_api_client_session_protocol_retry_and_cancellation(monkeypatch):[email protected]("finish", ["complete", "cancel", "poll_failure", "restart"])+async def test_compile_api_client_session_protocol_retry_and_cancellation(+ monkeypatch, tmp_path, finish+):forwarded = []response_status = {"cancel": "cancelled",@@ -415,100 +426,175 @@ async def test_compile_api_client_session_protocol_retry_and_cancellation(monkeyassert status_snapshot.meta == {"token_usage": {"total_tokens": 12}}assert cancel_snapshot.status == "cancelled"- class Tracker:+ # Exercise the real task store, work index, consumer, and ACK lifecycle.+ class QueueBackend:def __init__(self):- self.task = TaskRecord(- task_id="cmp_ov_1",- task_type="compile",- status=TaskStatus.RUNNING,- stage="compile: running",- account_id="acct",- user_id="alice",- meta={- "request": {- "from": ["viking://resources/source"],- "to": "viking://resources/wiki",- "skill": "viking://agent/skills/wiki",- },- "token_usage": {"total_tokens": 12},- },- )- self.auth = {- "openviking_connection": {"api_key": "active-user-key"},- "external_request_private": {},- }- self.stage = None- self.stage_updates = []- self.error = None-- async def get(self, *args, **kwargs):- return self.task-- async def get_task_auth(self, *args, **kwargs):- return self.auth-- async def start(self, *args, **kwargs):- return None-- async def update_task_auth(self, _task_id, values, **kwargs):- self.auth.update(values)-- async def update_stage(self, _task_id, stage, **kwargs):- self.stage = stage- self.stage_updates.append(stage)- self.task.stage = stage-- async def fail(self, _task_id, error, **kwargs):- self.error = error- self.task.status = TaskStatus.FAILED-- async def complete(self, _task_id, result, **kwargs):- self.task.result = result- self.task.status = TaskStatus.COMPLETED-- async def record_cancelled(self, _task_id, **kwargs):- self.task.status = TaskStatus.CANCELLED-- def is_cancellation_requested(self, _task_id):- return False-- async def no_sleep(_delay):- return None-- monkeypatch.setattr(external_task_service_module.asyncio, "sleep", no_sleep)- tracker = Tracker()+ self.pending = deque()+ self.processing = {}+ self.sequence = 0++ async def mkdir(self, path):+ pass++ async def write(self, path, data):+ if path.endswith("/enqueue"):+ self.sequence += 1+ message = {"id": str(self.sequence), "data": json.loads(data)}+ self.pending.append(message)+ return message["id"]+ assert path.endswith("/ack")+ del self.processing[data.decode()]++ async def read(self, path):+ if path.endswith("/messages"):+ result = [*self.pending, *self.processing.values()]+ elif path.endswith("/size"):+ result = len(self.pending)+ else:+ assert path.endswith("/dequeue")+ result = self.pending.popleft() if self.pending else {}+ if result:+ self.processing[result["id"]] = result+ return json.dumps(result).encode()++ backend = QueueBackend()+ manager = QueueManager(agfs=object())+ queue = manager.get_queue(+ QueueManager.EXTERNAL_TASK,+ dequeue_handler=ExternalTaskProcessor(tasks, asyncio.get_running_loop()),+ allow_create=True,+ )+ queue._async_agfs = backend+ store = PersistentTaskStore(MockLocalAGFS(root_path=tmp_path))+ tracker = TaskTracker(store)monkeypatch.setattr(external_task_service_module, "get_task_tracker", lambda: tracker)- forwarded.clear()- response_status["submit_failures"] = 1- response_status["poll"] = ["running", "completed"]-- await tasks.execute("cmp_ov_1", "acct", "alice")+ monkeypatch.setattr(external_task_service_module, "get_queue_manager", lambda: manager)+ monkeypatch.setattr(+ "openviking.storage.queuefs.external_task_processor.get_queue_manager", lambda: manager+ )+ await manager.prepare_task_tracking(tracker)+ real_sleep = asyncio.sleep++ async def yield_to_workers(_delay):+ await real_sleep(0)++ monkeypatch.setattr(external_task_service_module.asyncio, "sleep", yield_to_workers)+ entered = asyncio.Event()+ cancel_entered = asyncio.Event()+ release = asyncio.Event()+ submitted = []+ polls = 0+ rejected_id = None++ async def runtime_request(_client, method, url, headers, json):+ nonlocal polls+ if url.endswith("/runtime/v1/tasks"):+ task_id = headers["Idempotency-Key"]+ if task_id == rejected_id:+ return FakeResponse({"detail": "invalid request"}, status_code=422)+ submitted.append(task_id)+ return FakeResponse({"session_id": task_id})+ task_id = json["session_id"]+ if task_id != first.task_id:+ return FakeResponse({"status": "completed", "stage": "done", "result": {"ok": True}})+ if url.endswith("/cancel"):+ cancel_entered.set()+ await release.wait()+ return FakeResponse({"status": "cancelled", "stage": "cancelled"})+ polls += 1+ if polls == 1:+ # Runtime queueing is already submitted work, unlike OV queueing.+ return FakeResponse({"status": "pending", "stage": "queued"})+ entered.set()+ await release.wait()+ if finish == "poll_failure":+ return FakeResponse({"detail": "status unavailable"}, status_code=503)+ return FakeResponse({"status": "completed", "stage": "done", "result": {"ok": True}})++ monkeypatch.setattr(FakeClient, "request", runtime_request)++ async def create(user="alice", target="viking://resources/wiki", account="acct"):+ return await tasks.create(+ "compile",+ resource_id="viking://resources/source",+ payload={**public_payload, "to": target},+ connection={"api_key": "active-user-key"},+ ctx=RequestContext(user=UserIdentifier(account, user), role=Role.USER),+ )- assert [request["url"] for request in forwarded] == [- "https://compile.example.com/runtime/v1/tasks",- "https://compile.example.com/runtime/v1/tasks",- "https://compile.example.com/runtime/v1/tasks/status",- "https://compile.example.com/runtime/v1/tasks/status",- ]- assert forwarded[0]["headers"]["Idempotency-Key"] == "cmp_ov_1"- assert forwarded[1]["headers"]["Idempotency-Key"] == "cmp_ov_1"- assert tracker.stage_updates == ["compile: completed"]- assert tracker.task.status == TaskStatus.COMPLETED- assert tracker.task.result == {"output": "wiki"}-- forwarded.clear()- response_status["cancel"] = "cancelling"- response_status["cancel_failures"] = 3- response_status["poll"] = ["cancelling"] * 4 + ["cancelled"]- tracker.task.status = TaskStatus.CANCELLING- tracker.task.stage = "compile: running"- tracker.stage_updates.clear()-- await tasks.cancel_recovered("cmp_ov_1", "acct", "alice")-- assert tracker.task.status == TaskStatus.CANCELLED- assert response_status["cancel_failures"] == 0- assert response_status["poll"] == []+ async def state(task):+ return await tracker.get(task.task_id, account_id=task.account_id, user_id=task.user_id)++ first = await create()+ waiting = await create(user="bob")+ independent = await create(target="viking://resources/other")+ cancelled = await create()+ other_account = await create(account="other-account")+ worker = asyncio.create_task(queue.dequeue())+ await asyncio.wait_for(entered.wait(), timeout=5)++ await queue.dequeue() # Same target rotates, including across users.+ assert (await state(waiting)).status == TaskStatus.PENDING+ assert waiting.task_id not in submitted+ assert tracker.has_work(waiting.task_id) # Requeue survives ACK of the old delivery.+ await queue.dequeue()+ assert (await state(independent)).status == TaskStatus.COMPLETED+ await tracker.cancel(cancelled.task_id, account_id="acct", user_id="alice")+ await queue.dequeue()+ assert (await state(cancelled)).status == TaskStatus.CANCELLED+ assert cancelled.task_id not in submitted+ await queue.dequeue()+ assert (await state(other_account)).status == TaskStatus.COMPLETED++ if finish == "cancel":+ await tracker.cancel(first.task_id, account_id="acct", user_id="alice")+ await asyncio.wait_for(cancel_entered.wait(), timeout=5)+ assert (await state(first)).status == TaskStatus.CANCELLING+ elif finish == "restart":+ worker.cancel() # Process stops without requesting cancellation of the OV task.+ with pytest.raises(asyncio.CancelledError):+ await worker+ tracker = TaskTracker(store)+ tasks = ExternalTaskService()+ tasks.register(CompileService(service._config, tasks, SimpleNamespace()))+ queue.set_dequeue_handler(ExternalTaskProcessor(tasks, asyncio.get_running_loop()))+ await tasks.restore_tasks(await manager.prepare_task_tracking(tracker))+ # Recover processing behind pending work to prove recovery reserves the target first.+ backend.pending.extend(backend.processing.values())+ backend.processing.clear()++ await queue.dequeue()+ assert waiting.task_id not in submitted+ assert (await state(waiting)).status == TaskStatus.PENDING+ release.set()+ if finish == "restart":+ await queue.dequeue()+ else:+ await asyncio.wait_for(worker, timeout=5)+ expected = {+ "cancel": TaskStatus.CANCELLED,+ "poll_failure": TaskStatus.FAILED,+ }.get(finish, TaskStatus.COMPLETED)+ assert (await state(first)).status == expected+ if expected == TaskStatus.CANCELLED:+ # Recovery may replay a cancellation whose final ACK was lost.+ await tasks.cancel_recovered(first.task_id, "acct", "alice")+ if expected == TaskStatus.FAILED:+ assert "UNAVAILABLE" in (await state(first)).error+ assert not cancel_entered.is_set()+ await queue.dequeue()+ assert (await state(waiting)).status == TaskStatus.COMPLETED+ assert submitted.count(first.task_id) == 1+ assert not backend.pending and not backend.processing++ # Submission failure also releases the target for the next task.+ rejected = await create()+ rejected_id = rejected.task_id+ await queue.dequeue()+ assert (await state(rejected)).status == TaskStatus.FAILED+ after_rejection = await create()+ await queue.dequeue()+ assert (await state(after_rejection)).status == TaskStatus.COMPLETED@pytest.mark.asyncio