Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(compile): 同库同目标任务轮转排队 (#4863)

Qin Haojie39670e7b95a2
diff --git a/openviking/service/compile_service.py b/openviking/service/compile_service.py
index 6cb746c26..860264fa9 100644
--- a/openviking/service/compile_service.py
+++ b/openviking/service/compile_service.py
@@ -257,6 +257,9 @@ class CompileService:
def poll_interval_seconds(self) -> float:
return self._endpoint().poll_interval_ms / 1000.0
+ def serialization_key(self, payload: Mapping[str, Any]) -> str:
+ return payload["to"]
+
def configure_local_backend(self, base_url: str, gateway_token: str) -> None:
if self._config.base_url:
return
diff --git a/openviking/service/core.py b/openviking/service/core.py
index e1a1deac3..3d80f4861 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -526,7 +526,8 @@ class OpenVikingService:
self._queue_manager.USER_DELETION,
allow_create=True,
)
- await self._queue_manager.prepare_task_tracking(get_task_tracker())
+ restored_tasks = await self._queue_manager.prepare_task_tracking(get_task_tracker())
+ await self._external_task_service.restore_tasks(restored_tasks)
if self._config.enable_watch_scheduler:
await self._watch_scheduler.start()
diff --git a/openviking/service/external_task_service.py b/openviking/service/external_task_service.py
index c1689e05e..009b6359d 100644
--- a/openviking/service/external_task_service.py
+++ b/openviking/service/external_task_service.py
@@ -6,7 +6,7 @@ from __future__ import annotations
import asyncio
from dataclasses import dataclass
-from typing import Any, Awaitable, Callable, Mapping, Protocol
+from typing import Any, Awaitable, Callable, Iterable, Mapping, Protocol
from uuid import uuid4
from openviking.server.identity import RequestContext
@@ -51,6 +51,8 @@ class ExternalTaskProvider(Protocol):
@property
def poll_interval_seconds(self) -> float: ...
+ def serialization_key(self, payload: Mapping[str, Any]) -> str | None: ...
+
async def submit(
self,
ov_task_id: str,
@@ -77,6 +79,10 @@ class ExternalTaskService:
def __init__(self) -> None:
self._providers: dict[str, ExternalTaskProvider] = {}
+ # Accessed only on the service loop. Restored owners may include multiple
+ # tasks submitted before target serialization was enabled.
+ self._owners: dict[tuple[str | None, str, str], set[str]] = {}
+ self._executing: set[str] = set()
def register(self, provider: ExternalTaskProvider) -> None:
if provider.task_type in self._providers:
@@ -93,6 +99,36 @@ class ExternalTaskService:
)
return provider
+ def _serialization_key(self, task: TaskRecord) -> tuple[str | None, str, str] | None:
+ key = self._provider(task.task_type).serialization_key(task.meta["request"])
+ return (task.account_id, task.task_type, key) if key is not None else None
+
+ async def restore_tasks(self, tasks: Iterable[TaskRecord]) -> None:
+ """Reserve submitted work before any recovered queue delivery can run."""
+ for task in tasks:
+ if (
+ task.task_type not in self._providers
+ or task.result is not None
+ or task.error is not None
+ ):
+ continue
+ if task.status not in {TaskStatus.RUNNING, TaskStatus.CANCELLING}:
+ continue
+ if task.status == TaskStatus.CANCELLING and task.stage in {None, "queued"}:
+ auth = await self._task_auth(task.task_id, task.account_id, task.user_id)
+ if not auth.get("external_task_id"):
+ continue
+ key = self._serialization_key(task)
+ if key is not None:
+ self._owners.setdefault(key, set()).add(task.task_id)
+
+ def _release(self, task: TaskRecord) -> None:
+ key = self._serialization_key(task)
+ if key is not None and key in self._owners:
+ self._owners[key].discard(task.task_id)
+ if not self._owners[key]:
+ del self._owners[key]
+
async def create(
self,
task_type: str,
@@ -119,6 +155,12 @@ class ExternalTaskService:
)
enqueued = False
try:
+ await tracker.update_stage(
+ task.task_id,
+ "queued",
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ )
await get_queue_manager().enqueue(
QueueManager.EXTERNAL_TASK,
{
@@ -128,12 +170,6 @@ class ExternalTaskService:
},
)
enqueued = True
- await tracker.update_stage(
- task.task_id,
- "queued",
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- )
current = await tracker.get(
task.task_id,
account_id=ctx.account_id,
@@ -152,31 +188,67 @@ class ExternalTaskService:
raise
return task
- async def execute(self, task_id: str, account_id: str, user_id: str) -> None:
+ async def execute(self, task_id: str, account_id: str, user_id: str) -> bool:
+ """Return False when this delivery must rotate behind other target work."""
tracker = get_task_tracker()
- task = await tracker.get(task_id, account_id=account_id, user_id=user_id)
- if task is None or task.status in {
- TaskStatus.COMPLETED,
- TaskStatus.FAILED,
- TaskStatus.CANCELLED,
- }:
- return
- # The outcome is persisted before QueueFS ACK removes the owned work.
- # After a crash in that window, let the recovered delivery ACK without
- # resubmitting credentials that have already been cleared.
- if task.result is not None or task.error is not None:
- return
+ claimed = False
+ try:
+ task = await tracker.get(task_id, account_id=account_id, user_id=user_id)
+ if task is None or task.status in {
+ TaskStatus.COMPLETED,
+ TaskStatus.FAILED,
+ TaskStatus.CANCELLED,
+ }:
+ return True
+ # The outcome is persisted before QueueFS ACK removes the owned work.
+ # After a crash in that window, let the recovered delivery ACK without
+ # resubmitting credentials that have already been cleared.
+ if task.result is not None or task.error is not None:
+ return True
+
+ provider = self._provider(task.task_type)
+ payload = task.meta.get("request")
+ if not isinstance(payload, dict):
+ await tracker.fail(
+ task_id,
+ "INVALID_ARGUMENT: External task request is missing",
+ account_id=account_id,
+ user_id=user_id,
+ )
+ return True
+ key = self._serialization_key(task)
+ owners = self._owners.get(key, set()) if key is not None else set()
+ if task_id in self._executing or (owners and task_id not in owners):
+ return False
+ # No await between checking and claiming: concurrent deliveries on the
+ # service loop cannot both acquire an unoccupied target.
+ if key is not None:
+ self._owners.setdefault(key, set()).add(task_id)
+ self._executing.add(task_id)
+ claimed = True
+ await self._execute_task(task, provider, payload, account_id, user_id)
+ return True
+ except asyncio.CancelledError:
+ if not tracker.is_cancellation_requested(task_id):
+ raise
+ await run_to_completion(lambda: self.cancel_recovered(task_id, account_id, user_id))
+ raise
+ finally:
+ # An interrupted delivery keeps ownership until recovery confirms
+ # the Runtime has stopped; it must not admit another target writer.
+ if claimed:
+ self._executing.remove(task_id)
- provider = self._provider(task.task_type)
- payload = task.meta.get("request")
- if not isinstance(payload, dict):
- await tracker.fail(
- task_id,
- "INVALID_ARGUMENT: External task request is missing",
- account_id=account_id,
- user_id=user_id,
- )
- return
+ async def _execute_task(
+ self,
+ task: TaskRecord,
+ provider: ExternalTaskProvider,
+ payload: Mapping[str, Any],
+ account_id: str,
+ user_id: str,
+ ) -> None:
+ tracker = get_task_tracker()
+ task_id = task.task_id
auth = await self._task_auth(task_id, account_id, user_id)
connection = self._mapping(auth.get("openviking_connection"))
private_payload = self._mapping(auth.get("external_request_private"))
@@ -219,33 +291,24 @@ class ExternalTaskService:
return
await asyncio.sleep(provider.poll_interval_seconds)
except ExternalTaskError as exc:
- await tracker.fail(
- task_id,
- self._format_error(exc.code, str(exc)),
+ await self._apply_snapshot(
+ ExternalTaskSnapshot(status="failed", error_code=exc.code, error_message=str(exc)),
+ task_id=task_id,
account_id=account_id,
user_id=user_id,
)
- except asyncio.CancelledError:
- if not tracker.is_cancellation_requested(task_id):
- raise
- await run_to_completion(
- lambda: self._cancel_external(
- provider,
- ov_task_id=task_id,
- payload=payload,
- private_payload=private_payload,
- connection=connection,
- external_task_id=external_task_id,
- account_id=account_id,
- user_id=user_id,
- )
- )
- raise
async def cancel_recovered(self, task_id: str, account_id: str, user_id: str) -> None:
tracker = get_task_tracker()
task = await tracker.get(task_id, account_id=account_id, user_id=user_id)
- if task is None or task.stage in {None, "queued"}:
+ if task is None:
+ return
+ if (
+ task.status in {TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED}
+ or task.result is not None
+ or task.error is not None
+ ):
+ self._release(task)
return
payload = task.meta.get("request")
if not isinstance(payload, dict):
@@ -255,6 +318,9 @@ class ExternalTaskService:
transient=False,
)
auth = await self._task_auth(task_id, account_id, user_id)
+ if task.stage in {None, "queued"} and not auth.get("external_task_id"):
+ self._release(task)
+ return
await self._cancel_external(
self._provider(task.task_type),
ov_task_id=task_id,
@@ -287,12 +353,8 @@ class ExternalTaskService:
user_id: str,
) -> bool:
tracker = get_task_tracker()
+ task = await tracker.get(task_id, account_id=account_id, user_id=user_id)
if snapshot.stage is not None or snapshot.meta:
- task = await tracker.get(
- task_id,
- account_id=account_id,
- user_id=user_id,
- )
stage = snapshot.stage or snapshot.status
meta = snapshot.meta or {}
if task is not None and (
@@ -308,37 +370,42 @@ class ExternalTaskService:
)
if snapshot.status in _ACTIVE_STATUSES:
return False
- if snapshot.status == "completed":
- await tracker.complete(
- task_id,
- snapshot.result or snapshot.meta or {},
- account_id=account_id,
- user_id=user_id,
- )
- return True
- if snapshot.status == "failed":
- await tracker.fail(
- task_id,
- self._format_error(
- snapshot.error_code or "UNKNOWN",
- snapshot.error_message or "External task failed",
- ),
- account_id=account_id,
- user_id=user_id,
- )
- return True
- if snapshot.status == "cancelled":
- await tracker.record_cancelled(
- task_id,
- account_id=account_id,
- user_id=user_id,
+ if snapshot.status not in {"completed", "failed", "cancelled"}:
+ raise ExternalTaskError(
+ "INVALID_RESPONSE",
+ f"Unknown external task status: {snapshot.status}",
+ transient=False,
)
- return True
- raise ExternalTaskError(
- "INVALID_RESPONSE",
- f"Unknown external task status: {snapshot.status}",
- transient=False,
- )
+
+ async def finalize() -> None:
+ if snapshot.status == "failed":
+ await tracker.fail(
+ task_id,
+ self._format_error(
+ snapshot.error_code or "UNKNOWN",
+ snapshot.error_message or "External task failed",
+ ),
+ account_id=account_id,
+ user_id=user_id,
+ )
+ elif snapshot.status == "completed":
+ await tracker.complete(
+ task_id,
+ snapshot.result or snapshot.meta or {},
+ account_id=account_id,
+ user_id=user_id,
+ )
+ else:
+ await tracker.record_cancelled(
+ task_id,
+ account_id=account_id,
+ user_id=user_id,
+ )
+ if task is not None:
+ self._release(task)
+
+ await run_to_completion(finalize)
+ return True
async def _cancel_external(
self,
diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.py
index 3ea7911ef..a4b8c88db 100644
--- a/openviking/service/task_tracker.py
+++ b/openviking/service/task_tracker.py
@@ -214,10 +214,14 @@ class TaskTracker:
self._dispatcher.bind_current_loop()
self._install_work_index_callbacks()
- async def restore_work_tasks(self, owners: Dict[str, tuple[str, str]]) -> None:
+ async def restore_work_tasks(self, owners: Dict[str, tuple[str, str]]) -> List[TaskRecord]:
"""Restore task records referenced by rebuilt QueueFS work."""
+ restored = []
for task_id, (account_id, user_id) in owners.items():
- await self.get(task_id, account_id=account_id, user_id=user_id)
+ task = await self.get(task_id, account_id=account_id, user_id=user_id)
+ if task is not None:
+ restored.append(task)
+ return restored
async def _finalize_before_ack(self, metadata: QueueTaskMetadata) -> None:
"""Persist the terminal state before QueueFS removes the last recovery message."""
diff --git a/openviking/storage/queuefs/external_task_processor.py b/openviking/storage/queuefs/external_task_processor.py
index fc4031fea..2480e5f66 100644
--- a/openviking/storage/queuefs/external_task_processor.py
+++ b/openviking/storage/queuefs/external_task_processor.py
@@ -6,10 +6,12 @@ from __future__ import annotations
import asyncio
import json
-from typing import Any, Callable, Coroutine, Dict, Optional
+from typing import Any, Dict, Optional
from openviking.service.external_task_service import ExternalTaskService
from openviking.service.task_tracker_concurrency import OwnerLoopDispatcher
+from openviking.service.task_work_index import TaskWorkRejected
+from openviking.storage.queuefs import QueueManager, get_queue_manager
from openviking.storage.queuefs.named_queue import DequeueHandlerBase
@@ -38,12 +40,6 @@ class ExternalTaskProcessor(DequeueHandlerBase):
raise ValueError("External task queue payload is missing task ownership")
return task_id, account_id, user_id
- async def _run_on_service_loop(
- self,
- factory: Callable[[], Coroutine[Any, Any, None]],
- ) -> None:
- await self._dispatcher.run(factory)
-
async def on_dequeue(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
if not data:
return None
@@ -52,7 +48,22 @@ class ExternalTaskProcessor(DequeueHandlerBase):
except (json.JSONDecodeError, TypeError, ValueError) as exc:
self.report_error(str(exc), data)
return None
- await self._run_on_service_loop(lambda: self._service.execute(task_id, account_id, user_id))
+ processed = await self._dispatcher.run(
+ lambda: self._service.execute(task_id, account_id, user_id)
+ )
+ if not processed:
+ # Register a new delivery before the old one is ACKed. Keeping the
+ # task ID preserves cancellation and task-owned work across rotation.
+ try:
+ await get_queue_manager().enqueue(
+ QueueManager.EXTERNAL_TASK,
+ {"task_id": task_id, "account_id": account_id, "user_id": user_id},
+ )
+ except TaskWorkRejected:
+ # Cancellation only needs the current delivery to be ACKed.
+ pass
+ else:
+ self.report_requeue()
self.report_success()
return None
@@ -64,7 +75,7 @@ class ExternalTaskProcessor(DequeueHandlerBase):
except (json.JSONDecodeError, TypeError, ValueError) as exc:
self.report_error(str(exc), data)
return None
- await self._run_on_service_loop(
+ await self._dispatcher.run(
lambda: self._service.cancel_recovered(task_id, account_id, user_id)
)
self.report_success()
diff --git a/openviking/storage/queuefs/queue_manager.py b/openviking/storage/queuefs/queue_manager.py
index bf7e3e430..670e1a4b8 100644
--- a/openviking/storage/queuefs/queue_manager.py
+++ b/openviking/storage/queuefs/queue_manager.py
@@ -87,8 +87,8 @@ class QueueManager:
SESSION_COMMIT = "SessionCommit"
EXTERNAL_TASK = "ExternalTask"
USER_DELETION = "UserDeletion"
- # A deferred archive re-enqueues itself; throttle the next scheduling round.
- _SESSION_COMMIT_POLL_INTERVAL = 1.0
+ # Deferred work re-enqueues itself; throttle the next scheduling round.
+ _REQUEUE_POLL_INTERVAL = 1.0
def __init__(
self,
@@ -137,12 +137,12 @@ class QueueManager:
logger.info(f"[QueueManager] mount_point={self.mount_point} Started")
- async def prepare_task_tracking(self, tracker: Any) -> None:
+ async def prepare_task_tracking(self, tracker: Any) -> list[Any]:
"""Rebuild task work from QueueFS before any consumer starts."""
snapshots = {name: await queue.snapshot() for name, queue in self._queues.items()}
owners = self._task_work_index.rebuild(snapshots)
tracker.attach_work_index(self._task_work_index)
- await tracker.restore_work_tasks(owners)
+ return await tracker.restore_work_tasks(owners)
def setup_standard_queues(self, vector_store: Any, start: bool = True) -> None:
"""
@@ -225,8 +225,8 @@ class QueueManager:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
poll_interval = (
- self._SESSION_COMMIT_POLL_INTERVAL
- if queue.name == self.SESSION_COMMIT
+ self._REQUEUE_POLL_INTERVAL
+ if queue.name in {self.SESSION_COMMIT, self.EXTERNAL_TASK}
else self._poll_interval
)
try:
@@ -242,7 +242,7 @@ class QueueManager:
data = loop.run_until_complete(queue.dequeue())
if data is not None:
logger.debug("[QueueManager] Dequeued message from %s", queue.name)
- if queue.name == self.SESSION_COMMIT:
+ if queue.name in {self.SESSION_COMMIT, self.EXTERNAL_TASK}:
stop_event.wait(poll_interval)
else:
stop_event.wait(poll_interval)
@@ -261,8 +261,8 @@ class QueueManager:
A Semaphore caps inflight tasks at max_concurrent.
"""
poll_interval = (
- self._SESSION_COMMIT_POLL_INTERVAL
- if queue.name == self.SESSION_COMMIT
+ self._REQUEUE_POLL_INTERVAL
+ if queue.name in {self.SESSION_COMMIT, self.EXTERNAL_TASK}
else self._poll_interval
)
sem = asyncio.Semaphore(max_concurrent)
diff --git a/tests/server/test_bot_proxy_auth.py b/tests/server/test_bot_proxy_auth.py
index 80b3ad9c5..66d87a4f8 100644
--- a/tests/server/test_bot_proxy_auth.py
+++ b/tests/server/test_bot_proxy_auth.py
@@ -3,6 +3,9 @@
"""Regression tests for bot proxy endpoint auth enforcement."""
+import asyncio
+import json
+from collections import deque
from types import SimpleNamespace
import httpx
@@ -15,11 +18,16 @@ import openviking.service.compile_service as compile_service_module
import openviking.service.external_task_service as external_task_service_module
from openviking.server.auth.plugins import DevAuthPlugin, TrustedAuthPlugin
from openviking.server.config import ServerConfig
-from openviking.server.identity import AuthMode
+from openviking.server.identity import AuthMode, RequestContext, Role
from openviking.service.compile_service import CompileService
from openviking.service.external_task_service import ExternalTaskService
-from openviking.service.task_tracker import TaskRecord, TaskStatus
+from openviking.service.task_store import PersistentTaskStore
+from openviking.service.task_tracker import TaskRecord, TaskStatus, TaskTracker
+from openviking.storage.queuefs import QueueManager
+from openviking.storage.queuefs.external_task_processor import ExternalTaskProcessor
+from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils.config.open_viking_config import CompileApiConfig
+from tests.utils.mock_agfs import MockLocalAGFS
def test_set_bot_api_key_updates_module_state():
@@ -287,7 +295,10 @@ async def test_compile_route_uses_ov_owned_task_and_rejects_legacy_routes(monkey
@pytest.mark.asyncio
-async def test_compile_api_client_session_protocol_retry_and_cancellation(monkeypatch):
[email protected]("finish", ["complete", "cancel", "poll_failure", "restart"])
+async def test_compile_api_client_session_protocol_retry_and_cancellation(
+ monkeypatch, tmp_path, finish
+):
forwarded = []
response_status = {
"cancel": "cancelled",
@@ -415,100 +426,175 @@ async def test_compile_api_client_session_protocol_retry_and_cancellation(monkey
assert status_snapshot.meta == {"token_usage": {"total_tokens": 12}}
assert cancel_snapshot.status == "cancelled"
- class Tracker:
+ # Exercise the real task store, work index, consumer, and ACK lifecycle.
+ class QueueBackend:
def __init__(self):
- self.task = TaskRecord(
- task_id="cmp_ov_1",
- task_type="compile",
- status=TaskStatus.RUNNING,
- stage="compile: running",
- account_id="acct",
- user_id="alice",
- meta={
- "request": {
- "from": ["viking://resources/source"],
- "to": "viking://resources/wiki",
- "skill": "viking://agent/skills/wiki",
- },
- "token_usage": {"total_tokens": 12},
- },
- )
- self.auth = {
- "openviking_connection": {"api_key": "active-user-key"},
- "external_request_private": {},
- }
- self.stage = None
- self.stage_updates = []
- self.error = None
-
- async def get(self, *args, **kwargs):
- return self.task
-
- async def get_task_auth(self, *args, **kwargs):
- return self.auth
-
- async def start(self, *args, **kwargs):
- return None
-
- async def update_task_auth(self, _task_id, values, **kwargs):
- self.auth.update(values)
-
- async def update_stage(self, _task_id, stage, **kwargs):
- self.stage = stage
- self.stage_updates.append(stage)
- self.task.stage = stage
-
- async def fail(self, _task_id, error, **kwargs):
- self.error = error
- self.task.status = TaskStatus.FAILED
-
- async def complete(self, _task_id, result, **kwargs):
- self.task.result = result
- self.task.status = TaskStatus.COMPLETED
-
- async def record_cancelled(self, _task_id, **kwargs):
- self.task.status = TaskStatus.CANCELLED
-
- def is_cancellation_requested(self, _task_id):
- return False
-
- async def no_sleep(_delay):
- return None
-
- monkeypatch.setattr(external_task_service_module.asyncio, "sleep", no_sleep)
- tracker = Tracker()
+ self.pending = deque()
+ self.processing = {}
+ self.sequence = 0
+
+ async def mkdir(self, path):
+ pass
+
+ async def write(self, path, data):
+ if path.endswith("/enqueue"):
+ self.sequence += 1
+ message = {"id": str(self.sequence), "data": json.loads(data)}
+ self.pending.append(message)
+ return message["id"]
+ assert path.endswith("/ack")
+ del self.processing[data.decode()]
+
+ async def read(self, path):
+ if path.endswith("/messages"):
+ result = [*self.pending, *self.processing.values()]
+ elif path.endswith("/size"):
+ result = len(self.pending)
+ else:
+ assert path.endswith("/dequeue")
+ result = self.pending.popleft() if self.pending else {}
+ if result:
+ self.processing[result["id"]] = result
+ return json.dumps(result).encode()
+
+ backend = QueueBackend()
+ manager = QueueManager(agfs=object())
+ queue = manager.get_queue(
+ QueueManager.EXTERNAL_TASK,
+ dequeue_handler=ExternalTaskProcessor(tasks, asyncio.get_running_loop()),
+ allow_create=True,
+ )
+ queue._async_agfs = backend
+ store = PersistentTaskStore(MockLocalAGFS(root_path=tmp_path))
+ tracker = TaskTracker(store)
monkeypatch.setattr(external_task_service_module, "get_task_tracker", lambda: tracker)
- forwarded.clear()
- response_status["submit_failures"] = 1
- response_status["poll"] = ["running", "completed"]
-
- await tasks.execute("cmp_ov_1", "acct", "alice")
+ monkeypatch.setattr(external_task_service_module, "get_queue_manager", lambda: manager)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.external_task_processor.get_queue_manager", lambda: manager
+ )
+ await manager.prepare_task_tracking(tracker)
+ real_sleep = asyncio.sleep
+
+ async def yield_to_workers(_delay):
+ await real_sleep(0)
+
+ monkeypatch.setattr(external_task_service_module.asyncio, "sleep", yield_to_workers)
+ entered = asyncio.Event()
+ cancel_entered = asyncio.Event()
+ release = asyncio.Event()
+ submitted = []
+ polls = 0
+ rejected_id = None
+
+ async def runtime_request(_client, method, url, headers, json):
+ nonlocal polls
+ if url.endswith("/runtime/v1/tasks"):
+ task_id = headers["Idempotency-Key"]
+ if task_id == rejected_id:
+ return FakeResponse({"detail": "invalid request"}, status_code=422)
+ submitted.append(task_id)
+ return FakeResponse({"session_id": task_id})
+ task_id = json["session_id"]
+ if task_id != first.task_id:
+ return FakeResponse({"status": "completed", "stage": "done", "result": {"ok": True}})
+ if url.endswith("/cancel"):
+ cancel_entered.set()
+ await release.wait()
+ return FakeResponse({"status": "cancelled", "stage": "cancelled"})
+ polls += 1
+ if polls == 1:
+ # Runtime queueing is already submitted work, unlike OV queueing.
+ return FakeResponse({"status": "pending", "stage": "queued"})
+ entered.set()
+ await release.wait()
+ if finish == "poll_failure":
+ return FakeResponse({"detail": "status unavailable"}, status_code=503)
+ return FakeResponse({"status": "completed", "stage": "done", "result": {"ok": True}})
+
+ monkeypatch.setattr(FakeClient, "request", runtime_request)
+
+ async def create(user="alice", target="viking://resources/wiki", account="acct"):
+ return await tasks.create(
+ "compile",
+ resource_id="viking://resources/source",
+ payload={**public_payload, "to": target},
+ connection={"api_key": "active-user-key"},
+ ctx=RequestContext(user=UserIdentifier(account, user), role=Role.USER),
+ )
- assert [request["url"] for request in forwarded] == [
- "https://compile.example.com/runtime/v1/tasks",
- "https://compile.example.com/runtime/v1/tasks",
- "https://compile.example.com/runtime/v1/tasks/status",
- "https://compile.example.com/runtime/v1/tasks/status",
- ]
- assert forwarded[0]["headers"]["Idempotency-Key"] == "cmp_ov_1"
- assert forwarded[1]["headers"]["Idempotency-Key"] == "cmp_ov_1"
- assert tracker.stage_updates == ["compile: completed"]
- assert tracker.task.status == TaskStatus.COMPLETED
- assert tracker.task.result == {"output": "wiki"}
-
- forwarded.clear()
- response_status["cancel"] = "cancelling"
- response_status["cancel_failures"] = 3
- response_status["poll"] = ["cancelling"] * 4 + ["cancelled"]
- tracker.task.status = TaskStatus.CANCELLING
- tracker.task.stage = "compile: running"
- tracker.stage_updates.clear()
-
- await tasks.cancel_recovered("cmp_ov_1", "acct", "alice")
-
- assert tracker.task.status == TaskStatus.CANCELLED
- assert response_status["cancel_failures"] == 0
- assert response_status["poll"] == []
+ async def state(task):
+ return await tracker.get(task.task_id, account_id=task.account_id, user_id=task.user_id)
+
+ first = await create()
+ waiting = await create(user="bob")
+ independent = await create(target="viking://resources/other")
+ cancelled = await create()
+ other_account = await create(account="other-account")
+ worker = asyncio.create_task(queue.dequeue())
+ await asyncio.wait_for(entered.wait(), timeout=5)
+
+ await queue.dequeue() # Same target rotates, including across users.
+ assert (await state(waiting)).status == TaskStatus.PENDING
+ assert waiting.task_id not in submitted
+ assert tracker.has_work(waiting.task_id) # Requeue survives ACK of the old delivery.
+ await queue.dequeue()
+ assert (await state(independent)).status == TaskStatus.COMPLETED
+ await tracker.cancel(cancelled.task_id, account_id="acct", user_id="alice")
+ await queue.dequeue()
+ assert (await state(cancelled)).status == TaskStatus.CANCELLED
+ assert cancelled.task_id not in submitted
+ await queue.dequeue()
+ assert (await state(other_account)).status == TaskStatus.COMPLETED
+
+ if finish == "cancel":
+ await tracker.cancel(first.task_id, account_id="acct", user_id="alice")
+ await asyncio.wait_for(cancel_entered.wait(), timeout=5)
+ assert (await state(first)).status == TaskStatus.CANCELLING
+ elif finish == "restart":
+ worker.cancel() # Process stops without requesting cancellation of the OV task.
+ with pytest.raises(asyncio.CancelledError):
+ await worker
+ tracker = TaskTracker(store)
+ tasks = ExternalTaskService()
+ tasks.register(CompileService(service._config, tasks, SimpleNamespace()))
+ queue.set_dequeue_handler(ExternalTaskProcessor(tasks, asyncio.get_running_loop()))
+ await tasks.restore_tasks(await manager.prepare_task_tracking(tracker))
+ # Recover processing behind pending work to prove recovery reserves the target first.
+ backend.pending.extend(backend.processing.values())
+ backend.processing.clear()
+
+ await queue.dequeue()
+ assert waiting.task_id not in submitted
+ assert (await state(waiting)).status == TaskStatus.PENDING
+ release.set()
+ if finish == "restart":
+ await queue.dequeue()
+ else:
+ await asyncio.wait_for(worker, timeout=5)
+ expected = {
+ "cancel": TaskStatus.CANCELLED,
+ "poll_failure": TaskStatus.FAILED,
+ }.get(finish, TaskStatus.COMPLETED)
+ assert (await state(first)).status == expected
+ if expected == TaskStatus.CANCELLED:
+ # Recovery may replay a cancellation whose final ACK was lost.
+ await tasks.cancel_recovered(first.task_id, "acct", "alice")
+ if expected == TaskStatus.FAILED:
+ assert "UNAVAILABLE" in (await state(first)).error
+ assert not cancel_entered.is_set()
+ await queue.dequeue()
+ assert (await state(waiting)).status == TaskStatus.COMPLETED
+ assert submitted.count(first.task_id) == 1
+ assert not backend.pending and not backend.processing
+
+ # Submission failure also releases the target for the next task.
+ rejected = await create()
+ rejected_id = rejected.task_id
+ await queue.dequeue()
+ assert (await state(rejected)).status == TaskStatus.FAILED
+ after_rejection = await create()
+ await queue.dequeue()
+ assert (await state(after_rejection)).status == TaskStatus.COMPLETED
@pytest.mark.asyncio