Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Durable execution of asynchronous tasks owned by external services."""
from __future__ import annotations
import asyncio
import hashlib
import json
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, Iterable, Mapping, Protocol
from uuid import uuid4
from openviking.server.identity import RequestContext
from openviking.service.task_domain import TaskPatch
from openviking.service.task_tracker import TaskRecord, TaskStatus, get_task_tracker
from openviking.service.task_tracker_concurrency import (
KeyedAsyncLockPool,
OwnerLoopDispatcher,
run_to_completion,
)
from openviking.service.task_work_index import bind_task_context
from openviking.storage.errors import ResourceBusyError
from openviking.storage.queuefs import QueueManager, get_queue_manager
from openviking_cli.exceptions import ConflictError
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
_ACTIVE_STATUSES = frozenset({"pending", "running", "cancelling"})
class ExternalTaskError(RuntimeError):
"""A classified provider failure used by the durable retry loop."""
def __init__(self, code: str, message: str, *, transient: bool) -> None:
super().__init__(message)
self.code = code
self.transient = transient
@dataclass(frozen=True)
class ExternalTaskSnapshot:
"""Provider-independent state returned by an external task API."""
status: str
stage: str | None = None
result: dict[str, Any] | None = None
meta: dict[str, Any] | None = None
error_code: str | None = None
error_message: str | None = None
class ExternalTaskProvider(Protocol):
"""Adapter contract for one external asynchronous task type."""
task_type: str
task_id_prefix: str
poll_max_attempts: int
@property
def poll_interval_seconds(self) -> float: ...
def serialization_key(self, payload: Mapping[str, Any]) -> str | None: ...
async def submit(
self,
ov_task_id: str,
payload: Mapping[str, Any],
private_payload: Mapping[str, Any],
connection: Mapping[str, Any],
) -> str: ...
async def get(
self,
external_task_id: str,
connection: Mapping[str, Any],
*,
payload: Mapping[str, Any] | None = None,
private_payload: Mapping[str, Any] | None = None,
) -> ExternalTaskSnapshot: ...
async def cancel(
self,
external_task_id: str,
connection: Mapping[str, Any],
*,
payload: Mapping[str, Any] | None = None,
private_payload: Mapping[str, Any] | None = None,
) -> ExternalTaskSnapshot: ...
class ExternalTaskService:
"""Own OV task state while registered providers perform the actual work."""
def __init__(self) -> None:
self._providers: dict[str, ExternalTaskProvider] = {}
# Restored before workers start, then accessed only by the ExternalTask
# queue's loop. Restored owners may include multiple
# tasks submitted before target serialization was enabled.
self._owners: dict[tuple[str | None, str, str], set[str]] = {}
self._executing: set[str] = set()
self._submission_locks = KeyedAsyncLockPool[str]()
self._submission_dispatcher = OwnerLoopDispatcher()
def register(self, provider: ExternalTaskProvider) -> None:
if provider.task_type in self._providers:
raise ValueError(f"External task provider already registered: {provider.task_type}")
self._providers[provider.task_type] = provider
def _provider(self, task_type: str) -> ExternalTaskProvider:
provider = self._providers.get(task_type)
if provider is None:
raise ExternalTaskError(
"UNAVAILABLE",
f"External task provider is not registered: {task_type}",
transient=False,
)
return provider
def _serialization_key(self, task: TaskRecord) -> tuple[str | None, str, str] | None:
key = self._provider(task.task_type).serialization_key(task.meta["request"])
return (task.account_id, task.task_type, key) if key is not None else None
async def restore_tasks(self, tasks: Iterable[TaskRecord]) -> None:
"""Reserve submitted work before any recovered queue delivery can run."""
for task in tasks:
if (
task.task_type not in self._providers
or task.result is not None
or task.error is not None
):
continue
if task.status not in {TaskStatus.RUNNING, TaskStatus.CANCELLING}:
continue
if task.status == TaskStatus.CANCELLING and task.stage in {None, "queued"}:
auth = await self._task_auth(task.task_id, task.account_id, task.user_id)
if not auth.get("external_task_id"):
continue
key = self._serialization_key(task)
if key is not None:
self._owners.setdefault(key, set()).add(task.task_id)
def _release(self, task: TaskRecord) -> None:
key = self._serialization_key(task)
if key is not None and key in self._owners:
self._owners[key].discard(task.task_id)
if not self._owners[key]:
del self._owners[key]
async def create(
self,
task_type: str,
*,
resource_id: str | None,
payload: Mapping[str, Any],
private_payload: Mapping[str, Any] | None = None,
connection: Mapping[str, Any],
ctx: RequestContext,
idempotency_key: str | None = None,
submission_payload: Mapping[str, Any] | None = None,
) -> TaskRecord:
"""``submission_payload`` (default ``payload``) is what idempotent retries must match."""
async def create_task():
return await self._create(
task_type,
resource_id=resource_id,
payload=payload,
private_payload=private_payload,
connection=connection,
ctx=ctx,
idempotency_key=idempotency_key,
submission_payload=submission_payload,
)
if not idempotency_key:
return await create_task()
key = submission_task_id(ctx, task_type, "", idempotency_key)
async def serialized():
async with self._submission_locks.acquire(key):
return await run_to_completion(create_task)
return await self._submission_dispatcher.run(serialized)
async def _create(
self,
task_type: str,
*,
resource_id: str | None,
payload: Mapping[str, Any],
private_payload: Mapping[str, Any] | None = None,
connection: Mapping[str, Any],
ctx: RequestContext,
idempotency_key: str | None = None,
submission_payload: Mapping[str, Any] | None = None,
) -> TaskRecord:
provider = self._provider(task_type)
tracker = get_task_tracker()
request_hash = submission_request_hash(
payload if submission_payload is None else submission_payload, private_payload
)
token = uuid4().hex
task_id = (
submission_task_id(ctx, task_type, provider.task_id_prefix, idempotency_key)
if idempotency_key
else f"{provider.task_id_prefix}{token}"
)
task = await tracker.create(
task_type,
resource_id=resource_id,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
task_id=task_id,
meta={
"request": dict(payload),
**(
{"submission_hash": request_hash, "submission_token": token}
if idempotency_key
else {}
),
},
auth={
"openviking_connection": dict(connection),
"external_request_private": dict(private_payload or {}),
},
)
if idempotency_key and task.meta.get("submission_hash") != request_hash:
raise ConflictError("Idempotency-Key was already used with different parameters")
if idempotency_key and task.meta.get("submission_token") != token:
# Recover a creation interrupted before queue delivery. Duplicate
# deliveries are safe: execute claims each task before submitting.
await self._resume_submission(task, ctx)
return task
enqueued = False
try:
await tracker.patch_task(
task.task_id,
TaskPatch(stage="queued"),
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
with bind_task_context(task.task_id, ctx.account_id, ctx.user.user_id):
await get_queue_manager().enqueue(
QueueManager.EXTERNAL_TASK,
{
"task_id": task.task_id,
"account_id": ctx.account_id,
"user_id": ctx.user.user_id,
},
)
enqueued = True
current = await tracker.get(
task.task_id,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
if current is not None:
task = current
except BaseException:
if not enqueued:
await tracker.fail(
task.task_id,
"Failed to enqueue external task",
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
raise
return task
async def recover_submission(
self,
task_type: str,
payload: Mapping[str, Any],
private_payload: Mapping[str, Any],
ctx: RequestContext,
key: str,
) -> TaskRecord | None:
async def recover():
provider = self._provider(task_type)
task = await get_task_tracker().get(
submission_task_id(ctx, task_type, provider.task_id_prefix, key),
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
if task is None:
return None
if task.meta.get("submission_hash") != submission_request_hash(
payload, private_payload
):
raise ConflictError("Idempotency-Key was already used with different parameters")
await self._resume_submission(task, ctx)
return task
async def serialized():
async with self._submission_locks.acquire(submission_task_id(ctx, task_type, "", key)):
return await run_to_completion(recover)
return await self._submission_dispatcher.run(serialized)
async def _resume_submission(self, task: TaskRecord, ctx: RequestContext) -> None:
if task.status == TaskStatus.PENDING:
with bind_task_context(task.task_id, ctx.account_id, ctx.user.user_id):
await get_queue_manager().enqueue(
QueueManager.EXTERNAL_TASK,
{
"task_id": task.task_id,
"account_id": ctx.account_id,
"user_id": ctx.user.user_id,
},
)
async def execute(self, task_id: str, account_id: str, user_id: str) -> bool:
"""Return False when this delivery must rotate behind other target work."""
tracker = get_task_tracker()
claimed = False
try:
task = await tracker.get(task_id, account_id=account_id, user_id=user_id)
if task is None or task.status in {
TaskStatus.COMPLETED,
TaskStatus.FAILED,
TaskStatus.CANCELLED,
}:
return True
# The outcome is persisted before QueueFS ACK removes the owned work.
# After a crash in that window, let the recovered delivery ACK without
# resubmitting credentials that have already been cleared.
if task.result is not None or task.error is not None:
return True
provider = self._provider(task.task_type)
payload = task.meta.get("request")
if not isinstance(payload, dict):
await tracker.fail(
task_id,
"INVALID_ARGUMENT: External task request is missing",
account_id=account_id,
user_id=user_id,
)
return True
key = self._serialization_key(task)
owners = self._owners.get(key, set()) if key is not None else set()
if task_id in self._executing or (owners and task_id not in owners):
return False
# No await between checking and claiming: concurrent deliveries on the
# queue loop cannot both acquire an unoccupied target.
if key is not None:
self._owners.setdefault(key, set()).add(task_id)
self._executing.add(task_id)
claimed = True
on_start = getattr(provider, "on_start", None)
if on_start is not None:
started = await on_start(task)
if started == "busy":
self._release(task)
return False
if isinstance(started, ExternalTaskSnapshot):
await self._apply_snapshot(
started, task_id=task_id, account_id=account_id, user_id=user_id
)
return True
await self._execute_task(task, provider, payload, account_id, user_id)
return True
except asyncio.CancelledError:
if not await tracker.is_cancellation_requested(
task_id, account_id=account_id, user_id=user_id
):
raise
await run_to_completion(lambda: self.cancel_recovered(task_id, account_id, user_id))
raise
finally:
# An interrupted delivery keeps ownership until recovery confirms
# the Runtime has stopped; it must not admit another target writer.
if claimed:
self._executing.remove(task_id)
async def _execute_task(
self,
task: TaskRecord,
provider: ExternalTaskProvider,
payload: Mapping[str, Any],
account_id: str,
user_id: str,
) -> None:
tracker = get_task_tracker()
task_id = task.task_id
auth = await self._task_auth(task_id, account_id, user_id)
connection = self._mapping(auth.get("openviking_connection"))
private_payload = self._mapping(auth.get("external_request_private"))
external_task_id = str(auth.get("external_task_id") or "").strip() or None
try:
await tracker.start(
task_id,
account_id=account_id,
user_id=user_id,
stage="polling" if external_task_id else "submitting",
)
if external_task_id is None:
external_task_id = await self._retry(
lambda: provider.submit(task_id, payload, private_payload, connection),
task_id=task_id,
operation_name="submit",
poll_interval=provider.poll_interval_seconds,
max_attempts=provider.poll_max_attempts,
)
await tracker.patch_task(
task_id,
TaskPatch(auth={"external_task_id": external_task_id}),
account_id=account_id,
user_id=user_id,
)
while True:
snapshot = await self._retry(
lambda: provider.get(
external_task_id,
connection,
payload=payload,
private_payload=private_payload,
),
task_id=task_id,
operation_name="poll",
poll_interval=provider.poll_interval_seconds,
max_attempts=provider.poll_max_attempts,
)
if await self._apply_snapshot(
snapshot,
task_id=task_id,
account_id=account_id,
user_id=user_id,
):
return
await asyncio.sleep(provider.poll_interval_seconds)
except ExternalTaskError as exc:
await self._apply_snapshot(
ExternalTaskSnapshot(status="failed", error_code=exc.code, error_message=str(exc)),
task_id=task_id,
account_id=account_id,
user_id=user_id,
)
async def cancel_recovered(self, task_id: str, account_id: str, user_id: str) -> None:
tracker = get_task_tracker()
task = await tracker.get(task_id, account_id=account_id, user_id=user_id)
if task is None:
return
if (
task.status in {TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED}
or task.result is not None
or task.error is not None
):
self._release(task)
return
payload = task.meta.get("request")
if not isinstance(payload, dict):
raise ExternalTaskError(
"INVALID_ARGUMENT",
"External task request is missing",
transient=False,
)
auth = await self._task_auth(task_id, account_id, user_id)
if task.stage in {None, "queued"} and not auth.get("external_task_id"):
await self._apply_snapshot(
ExternalTaskSnapshot(status="cancelled"),
task_id=task_id,
account_id=account_id,
user_id=user_id,
)
return
await self._cancel_external(
self._provider(task.task_type),
ov_task_id=task_id,
payload=payload,
private_payload=self._mapping(auth.get("external_request_private")),
connection=self._mapping(auth.get("openviking_connection")),
external_task_id=str(auth.get("external_task_id") or "").strip() or None,
account_id=account_id,
user_id=user_id,
)
@staticmethod
async def _task_auth(task_id: str, account_id: str, user_id: str) -> dict[str, Any]:
return await get_task_tracker().get_task_auth(
task_id,
account_id=account_id,
user_id=user_id,
)
@staticmethod
def _mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, dict) else {}
async def _apply_snapshot(
self,
snapshot: ExternalTaskSnapshot,
*,
task_id: str,
account_id: str,
user_id: str,
) -> bool:
tracker = get_task_tracker()
task = await tracker.get(task_id, account_id=account_id, user_id=user_id)
if snapshot.stage is not None or snapshot.meta:
stage = snapshot.stage or snapshot.status
meta = snapshot.meta or {}
if task is not None and (
task.stage != stage
or any(
key not in task.meta or task.meta[key] != value for key, value in meta.items()
)
):
await tracker.patch_task(
task_id,
TaskPatch(stage=stage, meta=meta),
account_id=account_id,
user_id=user_id,
)
if snapshot.status in _ACTIVE_STATUSES:
return False
if snapshot.status not in {"completed", "failed", "cancelled"}:
raise ExternalTaskError(
"INVALID_RESPONSE",
f"Unknown external task status: {snapshot.status}",
transient=False,
)
if task is not None:
on_finish = getattr(self._providers.get(task.task_type), "on_finish", None)
if on_finish is not None:
async def finish():
"""Keep lock retries inside cancellation-protected provider finalization."""
while True:
try:
return await on_finish(task, snapshot)
except ResourceBusyError:
await asyncio.sleep(self._providers[task.task_type].poll_interval_seconds)
snapshot = await run_to_completion(finish)
async def finalize() -> None:
if snapshot.status == "failed":
await tracker.fail(
task_id,
self._format_error(
snapshot.error_code or "UNKNOWN",
snapshot.error_message or "External task failed",
),
account_id=account_id,
user_id=user_id,
)
elif snapshot.status == "completed":
await tracker.complete(
task_id,
snapshot.result or snapshot.meta or {},
account_id=account_id,
user_id=user_id,
)
else:
await tracker.record_cancelled(
task_id,
account_id=account_id,
user_id=user_id,
)
if task is not None:
self._release(task)
await run_to_completion(finalize)
return True
async def _cancel_external(
self,
provider: ExternalTaskProvider,
*,
ov_task_id: str,
payload: Mapping[str, Any],
private_payload: Mapping[str, Any],
connection: Mapping[str, Any],
external_task_id: str | None,
account_id: str,
user_id: str,
) -> None:
max_attempts = provider.poll_max_attempts
# Returning allows QueueFS to ACK the work and finalize cancellation.
# Keep ownership until the provider confirms a terminal state.
while True:
try:
if external_task_id is None:
# Submission may have reached the provider before local cancellation
# interrupted its response. Recover it with the same idempotency key.
external_task_id = await self._retry(
lambda: provider.submit(ov_task_id, payload, private_payload, connection),
task_id=ov_task_id,
operation_name="recover before cancel",
poll_interval=provider.poll_interval_seconds,
max_attempts=max_attempts,
)
await get_task_tracker().patch_task(
ov_task_id,
TaskPatch(auth={"external_task_id": external_task_id}),
account_id=account_id,
user_id=user_id,
)
snapshot = await self._retry(
lambda task_id=external_task_id: provider.cancel(
task_id, connection, payload=payload, private_payload=private_payload
),
task_id=ov_task_id,
operation_name="cancel",
poll_interval=provider.poll_interval_seconds,
max_attempts=max_attempts,
)
while True:
if await self._apply_snapshot(
snapshot,
task_id=ov_task_id,
account_id=account_id,
user_id=user_id,
):
return
await asyncio.sleep(provider.poll_interval_seconds)
snapshot = await self._retry(
lambda task_id=external_task_id: provider.get(
task_id, connection, payload=payload, private_payload=private_payload
),
task_id=ov_task_id,
operation_name="poll cancellation",
poll_interval=provider.poll_interval_seconds,
max_attempts=max_attempts,
)
except ExternalTaskError as exc:
logger.warning(
"External task cancellation will retry task=%s code=%s: %s",
ov_task_id,
exc.code,
exc,
)
await asyncio.sleep(provider.poll_interval_seconds)
@staticmethod
async def _retry(
action: Callable[[], Awaitable[Any]],
*,
task_id: str,
operation_name: str,
poll_interval: float,
max_attempts: int,
) -> Any:
delay = max(poll_interval, 0.2)
attempts = 0
while True:
try:
return await action()
except ExternalTaskError as exc:
if not exc.transient:
raise
attempts += 1
if attempts >= max_attempts:
raise
logger.warning(
"External task %s will retry task=%s code=%s: %s",
operation_name,
task_id,
exc.code,
exc,
)
await asyncio.sleep(delay)
delay = min(delay * 2, 30.0)
@staticmethod
def _format_error(code: str, message: str) -> str:
return f"{code}: {message}"
__all__ = [
"ExternalTaskError",
"ExternalTaskProvider",
"ExternalTaskService",
"ExternalTaskSnapshot",
]
def submission_task_id(ctx: RequestContext, task_type: str, prefix: str, key: str) -> str:
scope = json.dumps([ctx.account_id, ctx.user.user_id, task_type, key])
return prefix + hashlib.sha256(scope.encode()).hexdigest()
def submission_request_hash(
payload: Mapping[str, Any], private_payload: Mapping[str, Any] | None
) -> str:
return hashlib.sha256(
json.dumps(
{"payload": dict(payload), "private": dict(private_payload or {})},
sort_keys=True,
ensure_ascii=False,
separators=(",", ":"),
).encode()
).hexdigest()