Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(tasks): fence admission and preserve accepted task ownership (#6723)

runyunzhou2e8e53b28d8f
diff --git a/openviking/server/account_stores/file.py b/openviking/server/account_stores/file.py
index 01b5c6788..16c2853f6 100644
--- a/openviking/server/account_stores/file.py
+++ b/openviking/server/account_stores/file.py
@@ -15,6 +15,7 @@ from argon2 import PasswordHasher
from argon2.exceptions import VerificationError
from typing_extensions import deprecated
+from openviking.concurrency import AsyncSemaphore
from openviking.core.identifiers import validate_identifier_length
from openviking.pyagfs import AGFSAlreadyExistsError, AGFSNotFoundError, AsyncAGFSClient
from openviking.pyagfs.async_client import fs_ctx_from_agfs_path
@@ -135,11 +136,11 @@ class FileStore(AccountStore):
self._prefix_index: Dict[str, list[UserKeyEntry]] = {}
self._user_group_ids: Dict[tuple[str, str], tuple[str, ...]] = {}
# Serializes internal refreshes so overlapping reloads can't interleave.
- self._reload_lock = asyncio.Lock()
+ self._reload_lock = AsyncSemaphore(1)
# Serializes all mutations made by the unified manager in this process.
# File locks remain responsible for inter-instance coordination.
- self._mutation_lock = asyncio.Lock()
- self._deletion_lock = asyncio.Lock()
+ self._mutation_lock = AsyncSemaphore(1)
+ self._deletion_lock = AsyncSemaphore(1)
def _discard_account_state(self, account_id: str) -> None:
"""Remove an account and its key index entries from in-memory state."""
diff --git a/openviking/server/app.py b/openviking/server/app.py
index 5ee94688f..7809a6540 100644
--- a/openviking/server/app.py
+++ b/openviking/server/app.py
@@ -189,6 +189,7 @@ async def _initialize_runtime_state(
oauth_store=getattr(app.state, "oauth_store", None),
usage_audit_runtime=getattr(app.state, "usage_audit_runtime", None),
gateway_app=getattr(app.state, "gateway_app", None),
+ require_registered_identity=config.get_effective_auth_mode() == "api_key",
)
logger.info("OpenVikingService initialization complete")
diff --git a/openviking/service/deletion.py b/openviking/service/deletion.py
index f6b66253d..ad6d2d696 100644
--- a/openviking/service/deletion.py
+++ b/openviking/service/deletion.py
@@ -71,6 +71,7 @@ class DeletionService:
oauth_store: Any = None,
usage_audit_runtime: Any = None,
gateway_app: Any = None,
+ require_registered_identity: bool = True,
) -> None:
self._service = service
self._manager = manager
@@ -79,11 +80,16 @@ class DeletionService:
self._oauth_store = oauth_store
self._usage_audit_runtime = usage_audit_runtime
self._gateway_app = gateway_app
+ self._require_registered_identity = require_registered_identity
self._request_lock = asyncio.Lock()
async def initialize(self) -> None:
"""Bind the queue consumer and reconcile persisted deletion fences."""
self._service.viking_fs.set_deletion_guard(self._is_deleting)
+ creation_guard = (
+ self._creation_blocked if self._require_registered_identity else self._is_deleting
+ )
+ get_task_tracker().set_deletion_guard(creation_guard)
queue_manager = self._service._queue_manager
queue = queue_manager.get_queue(queue_manager.DATA_CLEANUP)
queued_task_ids = {
@@ -134,6 +140,15 @@ class DeletionService:
)
return deletion is not None
+ async def _creation_blocked(self, account_id: str, user_id: str) -> bool:
+ """Reject missing registered identities, also after their deletion fence is gone."""
+ if account_id == SYSTEM_TASK_ACCOUNT_ID:
+ return False
+ if await self._is_deleting(account_id, user_id):
+ return True
+ user = await self._manager.get_user(account_id, user_id)
+ return user is None
+
async def delete(
self,
account_id: str,
@@ -420,9 +435,16 @@ class DeletionService:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError("Timed out waiting for tasks to stop")
- await tracker.wait(
- task.task_id, account_id=task.account_id, user_id=task.user_id, timeout=remaining
- )
+ try:
+ await tracker.wait(
+ task.task_id,
+ account_id=task.account_id,
+ user_id=task.user_id,
+ timeout=remaining,
+ )
+ except KeyError:
+ # A Task created concurrently with this deletion withdraws itself.
+ continue
async def _delete_uploads(self, ctx: RequestContext) -> None:
viking_fs = self._service.viking_fs
@@ -513,6 +535,7 @@ async def setup_deletion(
oauth_store: Any = None,
usage_audit_runtime: Any = None,
gateway_app: Any = None,
+ require_registered_identity: bool = True,
) -> Optional[DeletionService]:
"""Create the deletion service after storage and authentication are ready."""
if manager is None or service.viking_fs is None or service._queue_manager is None:
@@ -524,6 +547,7 @@ async def setup_deletion(
oauth_store=oauth_store,
usage_audit_runtime=usage_audit_runtime,
gateway_app=gateway_app,
+ require_registered_identity=require_registered_identity,
)
await deletion_service.initialize()
return deletion_service
diff --git a/openviking/service/resource_memory_link_service.py b/openviking/service/resource_memory_link_service.py
index 04a029645..cefc906a8 100644
--- a/openviking/service/resource_memory_link_service.py
+++ b/openviking/service/resource_memory_link_service.py
@@ -338,7 +338,6 @@ class ResourceMemoryLinkService:
account_id=ctx.account_id,
user_id=ctx.user.user_id,
timeout=_RESOURCE_REASON_COMMIT_TIMEOUT_SECONDS,
- poll_interval=0.1,
)
if task.status == TaskStatus.COMPLETED:
return task.to_dict()
diff --git a/openviking/service/task_store.py b/openviking/service/task_store.py
index 7924cbb43..ad852acd3 100644
--- a/openviking/service/task_store.py
+++ b/openviking/service/task_store.py
@@ -112,6 +112,14 @@ class StoreOperationResult:
return cls(StoreOperationOutcome.REJECTED, code, message)
+@dataclass(frozen=True)
+class TaskCreationResult:
+ """The stored Task and whether this call inserted it."""
+
+ task: TaskRecord
+ created: bool
+
+
@dataclass(frozen=True)
class TaskDeletionBlocker:
task_id: str
@@ -150,7 +158,19 @@ class TaskWorkStore(Protocol):
@property
def work_recovery_mode(self) -> WorkRecoveryMode: ...
- async def create(self, task: TaskRecord, *, reject_if_active: bool = False) -> TaskRecord: ...
+ async def create(
+ self, task: TaskRecord, *, reject_if_active: bool = False
+ ) -> TaskCreationResult: ...
+
+ async def withdraw_unaccepted(
+ self, task_id: str, *, account_id: str, user_id: str
+ ) -> StoreOperationResult:
+ """Atomically delete an initial pending Task only while it has no Work.
+
+ Reject Tasks with execution progress or any retained Work, including
+ closed Work. Serialize this decision with Task and Work mutations.
+ """
+ ...
async def get_task(
self,
@@ -352,6 +372,17 @@ class TaskWorkStore(Protocol):
TaskStore = TaskWorkStore
+def _can_withdraw_task(task: TaskRecord, *, has_work: bool) -> bool:
+ return (
+ task.status is TaskStatus.PENDING
+ and not has_work
+ and task.stage is None
+ and task.result is None
+ and task.error is None
+ and task.processing_seconds is None
+ )
+
+
@dataclass
class _WorkSet:
works: Dict[str, WorkRecord]
@@ -392,7 +423,9 @@ class PersistentTaskStore(TaskWorkStore):
def work_recovery_mode(self) -> WorkRecoveryMode:
return WorkRecoveryMode.QUEUE_SNAPSHOT
- async def create(self, task: TaskRecord, *, reject_if_active: bool = False) -> TaskRecord:
+ async def create(
+ self, task: TaskRecord, *, reject_if_active: bool = False
+ ) -> TaskCreationResult:
if not task.account_id or not task.user_id:
raise ValueError("PersistentTaskStore requires account_id and user_id")
business_key = (task.account_id, task.user_id, task.task_type, task.resource_id)
@@ -410,14 +443,34 @@ class PersistentTaskStore(TaskWorkStore):
async with self._record_locks.acquire(owner):
existing = await self._load_task(task.task_id, task.account_id, task.user_id)
if existing is not None:
- return self._copy_task(existing, self._work_sets.get(owner))
+ return TaskCreationResult(
+ self._copy_task(existing, self._work_sets.get(owner)), created=False
+ )
stored = deepcopy(task)
append_task_events(stored, None)
await self._write_task(stored)
with self._lock:
self._task_cache[owner] = stored
self._work_sets[owner] = _WorkSet({})
- return deepcopy(stored)
+ return TaskCreationResult(deepcopy(stored), created=True)
+
+ async def withdraw_unaccepted(
+ self, task_id: str, *, account_id: str, user_id: str
+ ) -> StoreOperationResult:
+ if not account_id or not user_id:
+ raise ValueError("Task withdrawal requires account_id and user_id")
+ owner = (account_id, user_id, task_id)
+ async with self._record_locks.acquire(owner):
+ task = await self._load_task(task_id, account_id, user_id)
+ if task is None:
+ return StoreOperationResult.noop(StoreOperationCode.NOT_FOUND)
+ work_set = self._work_sets.get(owner)
+ if not _can_withdraw_task(task, has_work=bool(work_set and work_set.works)):
+ return StoreOperationResult.rejected(
+ StoreOperationCode.INVALID_STATE, "Task has already been used"
+ )
+ await self._delete_task_unlocked(owner, task_exists=True)
+ return StoreOperationResult.applied()
async def get_task(
self,
diff --git a/openviking/service/task_store_mysql.py b/openviking/service/task_store_mysql.py
index 9db121819..b3a04542f 100644
--- a/openviking/service/task_store_mysql.py
+++ b/openviking/service/task_store_mysql.py
@@ -42,15 +42,17 @@ from openviking.service.task_domain import (
sanitize_error,
summarize_works,
)
-from openviking.service.task_events import append_task_event
+from openviking.service.task_events import TaskEventHistory, append_task_event
from openviking.service.task_store import (
DeleteTasksResult,
StoreOperationCode,
StoreOperationResult,
TaskAlreadyActive,
+ TaskCreationResult,
TaskDeletionBlocker,
TaskStatsSnapshot,
TaskWorkStore,
+ _can_withdraw_task,
_deep_merge,
)
from openviking.service.task_store_mysql_schema import (
@@ -299,7 +301,7 @@ class MySQLTaskWorkStore(TaskWorkStore):
def _task_from_row(
row: Any,
works: Iterable[WorkRecord] = (),
- execution_events: Optional[dict] = None,
+ execution_events: Optional[TaskEventHistory] = None,
) -> TaskRecord:
task = TaskRecord(
task_id=row.task_id,
@@ -418,7 +420,9 @@ class MySQLTaskWorkStore(TaskWorkStore):
rows = (await session.scalars(statement)).all()
return [self._work_from_row(row) for row in rows]
- async def _event_history(self, session: AsyncSession, task_id: str) -> Optional[dict]:
+ async def _event_history(
+ self, session: AsyncSession, task_id: str
+ ) -> Optional[TaskEventHistory]:
Event = self._models.Event
rows = (
await session.scalars(
@@ -445,36 +449,27 @@ class MySQLTaskWorkStore(TaskWorkStore):
"started_mid_task": bool(latest.started_mid_task),
}
- async def _store_event_history(
+ async def _append_event_history(
self,
session: AsyncSession,
task_id: str,
- history: Optional[dict],
+ history: Optional[TaskEventHistory],
+ previous: Optional[TaskEventHistory] = None,
) -> None:
+ """Insert new events under the Task lock; retained rows remain immutable."""
if history is None:
return
Event = self._models.Event
items = history["items"]
if not items:
return
- seqs = [int(item["seq"]) for item in items]
- existing = set(
- await session.scalars(
- select(Event.seq).where(
- *self._event_query(Event, task_id),
- Event.seq.in_(seqs),
- )
- )
- )
+ previous_seq = previous["items"][-1]["seq"] if previous and previous["items"] else 0
metadata = {
"dropped_count": int(history["dropped_count"]),
"started_mid_task": bool(history["started_mid_task"]),
}
- await session.execute(
- update(Event).where(*self._event_query(Event, task_id)).values(**metadata)
- )
for item in items:
- if int(item["seq"]) in existing:
+ if item["seq"] <= previous_seq:
continue
session.add(
Event(
@@ -490,9 +485,10 @@ class MySQLTaskWorkStore(TaskWorkStore):
**metadata,
)
)
- await session.execute(
- delete(Event).where(*self._event_query(Event, task_id), Event.seq < min(seqs))
- )
+ if history["dropped_count"] > (previous["dropped_count"] if previous else 0):
+ await session.execute(
+ delete(Event).where(*self._event_query(Event, task_id), Event.seq < items[0]["seq"])
+ )
async def _lock_work(self, session: AsyncSession, task_id: str, work_id: str) -> Any:
Work = self._models.Work
@@ -508,7 +504,9 @@ class MySQLTaskWorkStore(TaskWorkStore):
async with session.begin():
yield session
- async def create(self, task: TaskRecord, *, reject_if_active: bool = False) -> TaskRecord:
+ async def create(
+ self, task: TaskRecord, *, reject_if_active: bool = False
+ ) -> TaskCreationResult:
account_id, user_id = self._require_owner(task.account_id, task.user_id)
Task = self._models.Task
now = time.time()
@@ -560,12 +558,15 @@ class MySQLTaskWorkStore(TaskWorkStore):
raise ValueError(
f"Task id already belongs to another owner: {stored.task_id}"
)
- return self._task_from_row(
- existing,
- execution_events=await self._event_history(session, stored.task_id),
+ return TaskCreationResult(
+ self._task_from_row(
+ existing,
+ execution_events=await self._event_history(session, stored.task_id),
+ ),
+ created=False,
)
session.add(row)
- await self._store_event_history(session, stored.task_id, stored.execution_events)
+ await self._append_event_history(session, stored.task_id, stored.execution_events)
except IntegrityError as exc:
if not self._is_duplicate_error(exc):
raise
@@ -581,14 +582,17 @@ class MySQLTaskWorkStore(TaskWorkStore):
raise ValueError(
f"Task id already belongs to another owner: {stored.task_id}"
)
- return self._task_from_row(
- existing,
- execution_events=await self._event_history(session, stored.task_id),
+ return TaskCreationResult(
+ self._task_from_row(
+ existing,
+ execution_events=await self._event_history(session, stored.task_id),
+ ),
+ created=False,
)
if reject_if_active:
raise TaskAlreadyActive(stored.task_type, stored.resource_id) from exc
raise
- return stored
+ return TaskCreationResult(stored, created=True)
@staticmethod
def _is_duplicate_error(exc: IntegrityError) -> bool:
@@ -776,6 +780,34 @@ class MySQLTaskWorkStore(TaskWorkStore):
)
)
+ async def withdraw_unaccepted(
+ self, task_id: str, *, account_id: str, user_id: str
+ ) -> StoreOperationResult:
+ self._require_owner(account_id, user_id)
+ Work = self._models.Work
+ async with self._transaction() as session:
+ row = await self._load_task(
+ session, task_id, account_id=account_id, user_id=user_id, lock=True
+ )
+ if row is None:
+ return StoreOperationResult.noop(StoreOperationCode.NOT_FOUND)
+ has_work = bool(
+ await session.scalar(
+ select(
+ exists().where(
+ Work.service_name == self._service_name,
+ Work.task_id == task_id,
+ )
+ )
+ )
+ )
+ if not _can_withdraw_task(self._task_from_row(row), has_work=has_work):
+ return StoreOperationResult.rejected(
+ StoreOperationCode.INVALID_STATE, "Task has already been used"
+ )
+ await self._delete_task_records(session, [task_id])
+ return StoreOperationResult.applied()
+
async def delete_tasks(
self,
task_id: Optional[str] = None,
@@ -969,7 +1001,9 @@ class MySQLTaskWorkStore(TaskWorkStore):
active_business_key=None,
)
)
- await self._store_event_history(session, task_id, task.execution_events)
+ await self._append_event_history(
+ session, task_id, task.execution_events, previous.execution_events
+ )
return task
async def start_task(
@@ -1014,7 +1048,9 @@ class MySQLTaskWorkStore(TaskWorkStore):
updated_at=TaskRecord.next_updated_at(row.updated_at),
)
)
- await self._store_event_history(session, task_id, task.execution_events)
+ await self._append_event_history(
+ session, task_id, task.execution_events, previous.execution_events
+ )
return StoreOperationResult.applied()
async def patch_task(
@@ -1070,7 +1106,9 @@ class MySQLTaskWorkStore(TaskWorkStore):
task = deepcopy(previous)
task.stage = stage
append_task_events(task, previous)
- await self._store_event_history(session, task_id, task.execution_events)
+ await self._append_event_history(
+ session, task_id, task.execution_events, previous.execution_events
+ )
if "meta" in patch.clear_fields or patch.meta is not None:
meta = {}
if "meta" not in patch.clear_fields:
@@ -1126,8 +1164,9 @@ class MySQLTaskWorkStore(TaskWorkStore):
StoreOperationCode.ALREADY_TERMINAL,
f"Task is already {row.status}",
)
+ previous = await self._event_history(session, task_id)
events = append_task_event(
- await self._event_history(session, task_id),
+ deepcopy(previous),
kind=kind,
status=row.status,
stage=sanitize_error(row.stage) if row.stage else None,
@@ -1140,7 +1179,7 @@ class MySQLTaskWorkStore(TaskWorkStore):
updated_at=TaskRecord.next_updated_at(row.updated_at),
)
)
- await self._store_event_history(session, task_id, events)
+ await self._append_event_history(session, task_id, events, previous)
return StoreOperationResult.applied()
async def record_outcome(
@@ -1204,6 +1243,7 @@ class MySQLTaskWorkStore(TaskWorkStore):
next_status = TaskStatus.CANCELLING if cancelled else TaskStatus(row.status)
events = None
+ previous_events = None
if cancelled or (error and row.error is None):
previous = TaskRecord(
task_id=task_id,
@@ -1213,6 +1253,7 @@ class MySQLTaskWorkStore(TaskWorkStore):
error=row.error,
execution_events=await self._event_history(session, task_id),
)
+ previous_events = previous.execution_events
task = deepcopy(previous)
task.status = next_status
task.error = next_error
@@ -1236,7 +1277,7 @@ class MySQLTaskWorkStore(TaskWorkStore):
updated_at=TaskRecord.next_updated_at(row.updated_at),
)
)
- await self._store_event_history(session, task_id, events)
+ await self._append_event_history(session, task_id, events, previous_events)
finalized = await self._finalize_if_closed(
session,
task_id,
@@ -1321,7 +1362,9 @@ class MySQLTaskWorkStore(TaskWorkStore):
updated_at=TaskRecord.next_updated_at(row.updated_at),
)
)
- await self._store_event_history(session, task_id, task.execution_events)
+ await self._append_event_history(
+ session, task_id, task.execution_events, previous.execution_events
+ )
finalized = await self._finalize_if_closed(
session,
task_id,
diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.py
index 5a8c3e694..ccd442904 100644
--- a/openviking/service/task_tracker.py
+++ b/openviking/service/task_tracker.py
@@ -54,12 +54,17 @@ from openviking.service.task_store import (
StoreOperationCode,
StoreOperationResult,
TaskAlreadyActive,
+ TaskCreationResult,
TaskWorkStore,
WorkRegistration,
)
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.service.task_work_index import TaskWorkIndex
-from openviking_cli.exceptions import DeadlineExceededError, OpenVikingError
+from openviking_cli.exceptions import (
+ DeadlineExceededError,
+ FailedPreconditionError,
+ OpenVikingError,
+)
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
@@ -85,8 +90,10 @@ _CANCELLABLE_TASK_TYPES = {
"admin_reindex",
"snapshot_restore_reindex",
}
+_IDENTITY_CLEANUP_TASK_TYPES = {"account_delete", "user_delete"}
_CANCEL_POLL_INTERVAL = 30
-_TASK_WAIT_POLL_INTERVAL = 0.05
+_WAIT_INITIAL_POLL_INTERVAL = 0.05
+_WAIT_MAX_POLL_INTERVAL = 10.0
_PUBLIC_QUEUE_NAMES = ("Semantic", "Embedding")
_sanitize_error = sanitize_error
@@ -102,6 +109,28 @@ class _WaitBudget:
_wait_budget: ContextVar[Optional[_WaitBudget]] = ContextVar("task_wait_budget", default=None)
+_admission: ContextVar[Optional[tuple[str, Callable[[], Awaitable[bool]]]]] = ContextVar(
+ "task_admission", default=None
+)
+
+
+@contextmanager
+def task_admission(
+ task_id: str, admitted: Callable[[], Awaitable[bool]]
+) -> Generator[None, None, None]:
+ """Create *task_id* within this context only while *admitted* holds.
+
+ create checks before the insert and again after it. A confirmed revocation
+ withdraws an unused Task or cancels one that another caller has taken over.
+ A failed check or cancelled caller only withdraws an unused Task.
+ """
+ token = _admission.set((task_id, admitted))
+ try:
+ yield
+ finally:
+ _admission.reset(token)
+
+
@contextmanager
def _consume_wait_budget(
task_id: str, timeout: Optional[float]
@@ -296,12 +325,52 @@ class TaskTracker:
self._running: ContextVar[tuple[str, str, str, bool] | None] = ContextVar(
"task_tracker_running_work", default=None
)
+ self._deletion_guard: Optional[Callable[[str, str], Any]] = None
@staticmethod
def _validate_owner(account_id: str, user_id: str) -> None:
if not account_id or not user_id:
raise ValueError("Task ownership requires non-empty account_id and user_id")
+ def set_deletion_guard(self, guard: Optional[Callable[[str, str], Any]]) -> None:
+ """Reject new Tasks for identities the guard reports as being deleted.
+
+ Identity cleanup records its fence and then lists the Tasks to cancel;
+ create inserts the Task and then reads the fence again. One of the two
+ reads observes the other's write, so a Task inserted concurrently with
+ a deletion is either listed by cleanup or withdrawn/cancelled here.
+ """
+ self._deletion_guard = guard
+
+ async def _owner_deleting(self, account_id: str, user_id: str) -> bool:
+ guard = self._deletion_guard
+ if guard is None or account_id == SYSTEM_TASK_ACCOUNT_ID:
+ return False
+ result = guard(account_id, user_id)
+ return bool(await result if inspect.isawaitable(result) else result)
+
+ async def _withdraw(
+ self, task_id: str, *, account_id: str, user_id: str, refusal: Optional[str]
+ ) -> None:
+ withdrawn = await self._store.withdraw_unaccepted(
+ task_id, account_id=account_id, user_id=user_id
+ )
+ if withdrawn.is_applied:
+ self._work_index.forget_task(task_id, account_id=account_id, user_id=user_id)
+ return
+ if withdrawn.accepted or refusal is None:
+ return
+ cancellation = await self._request_cancellation(
+ task_id,
+ account_id=account_id,
+ user_id=user_id,
+ allow_terminal=False,
+ )
+ if cancellation.accepted:
+ self._work_index.cancel_active(task_id, account_id=account_id, user_id=user_id)
+ elif cancellation.code is StoreOperationCode.NOT_CANCELLABLE:
+ await self.fail(task_id, refusal, account_id=account_id, user_id=user_id)
+
def attach_work_index(self, work_index: TaskWorkIndex) -> None:
"""Replace the process-local active-handle index."""
self._work_index = work_index
@@ -417,6 +486,15 @@ class TaskTracker:
existing = await self.get(task_id, account_id=account_id, user_id=user_id)
if existing is not None:
return existing
+ admission = _admission.get()
+ admitted = admission[1] if admission and admission[0] == task_id else None
+ # Identity cleanup may be owned by an actor that is itself being deleted;
+ # fencing it would strand the target deletion without a cleanup task.
+ enforce_identity_admission = task_type not in _IDENTITY_CLEANUP_TASK_TYPES
+ if enforce_identity_admission and await self._owner_deleting(account_id, user_id):
+ raise FailedPreconditionError("Identity deletion is in progress")
+ if admitted is not None and not await admitted():
+ raise FailedPreconditionError("Task creation is no longer admitted")
task = TaskRecord(
task_id=task_id or str(uuid4()),
task_type=task_type,
@@ -426,10 +504,41 @@ class TaskTracker:
meta=dict(meta or {}),
auth=dict(auth or {}),
)
- stored = await run_to_completion(
- lambda: self._store.create(task, reject_if_active=reject_if_active)
- )
- return self._public(stored or task)
+ creation: Optional[TaskCreationResult] = None
+ refusal: Optional[str] = None
+
+ async def persist() -> None:
+ nonlocal creation
+ # Retain the committed result even when the caller cancels during I/O.
+ creation = await self._store.create(task, reject_if_active=reject_if_active)
+
+ try:
+ await run_to_completion(persist)
+ assert creation is not None
+ created = self._public(creation.task)
+ if not creation.created:
+ return created
+ if enforce_identity_admission and await self._owner_deleting(account_id, user_id):
+ refusal = "Identity deletion is in progress"
+ raise FailedPreconditionError(refusal)
+ if admitted is not None and not await admitted():
+ refusal = "Task creation is no longer admitted"
+ raise FailedPreconditionError(refusal)
+ except BaseException:
+ if creation is not None and creation.created:
+ try:
+ await run_to_completion(
+ lambda: self._withdraw(
+ task.task_id,
+ account_id=account_id,
+ user_id=user_id,
+ refusal=refusal,
+ )
+ )
+ except BaseException:
+ logger.exception("[TaskTracker] Failed to withdraw Task %s", task.task_id)
+ raise
+ return created
async def create_if_no_running(
self,
@@ -743,10 +852,10 @@ class TaskTracker:
account_id: Optional[str] = None,
user_id: Optional[str] = None,
timeout: Optional[float] = None,
- poll_interval: float = _TASK_WAIT_POLL_INTERVAL,
) -> TaskRecord:
with _consume_wait_budget(task_id, timeout) as remaining:
deadline = None if remaining is None else time.monotonic() + remaining
+ next_poll_interval = _WAIT_INITIAL_POLL_INTERVAL
while True:
if account_id is not None and user_id is not None:
status = await self.get_task_status(
@@ -775,8 +884,11 @@ class TaskTracker:
if remaining is not None and remaining <= 0:
raise TimeoutError(f"Timed out waiting for Task {task_id}")
await asyncio.sleep(
- poll_interval if remaining is None else min(poll_interval, remaining)
+ next_poll_interval
+ if remaining is None
+ else min(next_poll_interval, remaining)
)
+ next_poll_interval = min(_WAIT_MAX_POLL_INTERVAL, next_poll_interval * 2)
async def list_tasks(
self,
@@ -1108,13 +1220,13 @@ class TaskTracker:
user_id: Optional[str] = None,
timeout: Optional[float] = None,
stop_waiting: Optional[Callable[[], bool]] = None,
- poll_interval: float = _TASK_WAIT_POLL_INTERVAL,
) -> None:
wanted = set(work_ids)
if not wanted:
return
with _consume_wait_budget(task_id, timeout) as remaining:
deadline = None if remaining is None else time.monotonic() + remaining
+ next_poll_interval = _WAIT_INITIAL_POLL_INTERVAL
while True:
works = await self._store.get_works(
task_id,
@@ -1133,8 +1245,11 @@ class TaskTracker:
if remaining is not None and remaining <= 0:
raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")
await asyncio.sleep(
- poll_interval if remaining is None else min(poll_interval, remaining)
+ next_poll_interval
+ if remaining is None
+ else min(next_poll_interval, remaining)
)
+ next_poll_interval = min(_WAIT_MAX_POLL_INTERVAL, next_poll_interval * 2)
async def wait_for_task_work(
self,
@@ -1144,10 +1259,10 @@ class TaskTracker:
account_id: Optional[str] = None,
user_id: Optional[str] = None,
timeout: Optional[float] = None,
- poll_interval: float = _TASK_WAIT_POLL_INTERVAL,
) -> None:
with _consume_wait_budget(task_id, timeout) as remaining:
deadline = None if remaining is None else time.monotonic() + remaining
+ next_poll_interval = _WAIT_INITIAL_POLL_INTERVAL
while await self._store.has_open_work(
task_id,
exclude_work_id=exclude_work_id,
@@ -1158,8 +1273,11 @@ class TaskTracker:
if remaining is not None and remaining <= 0:
raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")
await asyncio.sleep(
- poll_interval if remaining is None else min(poll_interval, remaining)
+ next_poll_interval
+ if remaining is None
+ else min(next_poll_interval, remaining)
)
+ next_poll_interval = min(_WAIT_MAX_POLL_INTERVAL, next_poll_interval * 2)
async def is_enqueue_complete(
self,
diff --git a/openviking/storage/mysql_runtime.py b/openviking/storage/mysql_runtime.py
index 7921199b4..4090a6246 100644
--- a/openviking/storage/mysql_runtime.py
+++ b/openviking/storage/mysql_runtime.py
@@ -247,7 +247,7 @@ def mysql_runtime_store(cls: type[T]) -> type[T]:
runtime: MySQLRuntime = self._mysql_runtime
async def invoke():
- if _name in ("load", "close"):
+ if _name in ("load", "initialize", "close"):
loop = asyncio.get_running_loop()
lifecycle = getattr(self, "_mysql_lifecycle_lock", None)
if lifecycle is None or lifecycle[0] is not loop:
diff --git a/tests/server/test_api_key_manager.py b/tests/server/test_api_key_manager.py
index 90b7606eb..b7c5b1e4d 100644
--- a/tests/server/test_api_key_manager.py
+++ b/tests/server/test_api_key_manager.py
@@ -5,6 +5,7 @@
import asyncio
import hashlib
+import threading
import uuid
from types import SimpleNamespace
@@ -103,6 +104,61 @@ async def test_create_duplicate_account_raises(manager: APIKeyManager):
await manager.create_account(acct, "bob")
+async def test_account_reads_and_mutations_cross_loops_after_waiter_cancellation(
+ manager: APIKeyManager, manager_service, monkeypatch
+):
+ acct = _uid()
+ await manager.create_account(acct, "admin")
+ agfs = manager_service.viking_fs.agfs
+ entered, release, readers_waiting = threading.Event(), threading.Event(), threading.Event()
+ write = agfs.write
+
+ def delayed_write(path, *args, **kwargs):
+ if path == f"/local/{acct}/_system/users.json" and not entered.is_set():
+ entered.set()
+ if not release.wait(timeout=10):
+ raise TimeoutError("account write was not released")
+ return write(path, *args, **kwargs)
+
+ async def worker_operations():
+ tasks = [
+ asyncio.create_task(manager.get_account(acct)),
+ asyncio.create_task(manager.get_user_membership(acct, "alice")),
+ asyncio.create_task(manager.register_user(acct, "bob")),
+ ]
+ await asyncio.sleep(0)
+ readers_waiting.set()
+ return await asyncio.wait_for(asyncio.gather(*tasks), timeout=5)
+
+ monkeypatch.setattr(agfs, "write", delayed_write)
+ writer = asyncio.create_task(manager.register_user(acct, "alice"))
+ worker = None
+ try:
+ assert await asyncio.to_thread(entered.wait, 5)
+ worker = asyncio.create_task(asyncio.to_thread(lambda: asyncio.run(worker_operations())))
+ assert await asyncio.to_thread(readers_waiting.wait, 5)
+ cancelled = asyncio.create_task(manager.get_account(acct))
+ await asyncio.sleep(0)
+ cancelled.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await cancelled
+ finally:
+ release.set()
+ results = await asyncio.gather(
+ writer, *([worker] if worker is not None else []), return_exceptions=True
+ )
+ assert not any(isinstance(result, BaseException) for result in results)
+ account, membership, bob_key = results[1]
+ assert account["status"] == "active"
+ assert membership == {"user_id": "alice", "role": Role.USER}
+ assert (await manager.resolve_identity(bob_key)).user_id == "bob"
+ assert {item["user_id"] for item in await _list_users(manager, acct)} == {
+ "admin",
+ "alice",
+ "bob",
+ }
+
+
async def test_ensure_trusted_identities_creates_missing_users_once(
manager: APIKeyManager,
):
diff --git a/tests/server/test_server_health.py b/tests/server/test_server_health.py
index 6608c76ad..9f0bf9329 100644
--- a/tests/server/test_server_health.py
+++ b/tests/server/test_server_health.py
@@ -104,7 +104,17 @@ async def test_health_endpoint_without_api_key():
@pytest.mark.parametrize(
- "credentials", ["anonymous", "partial", "trusted", "keyed", "malformed", "asserted-root", "root-only", "invalid-root-only"]
+ "credentials",
+ [
+ "anonymous",
+ "partial",
+ "trusted",
+ "keyed",
+ "malformed",
+ "asserted-root",
+ "root-only",
+ "invalid-root-only",
+ ],
)
async def test_health_trusted_identity_contract(credentials, caplog, service):
app = create_app(
@@ -288,10 +298,6 @@ async def test_ready_returns_200_after_initialized(monkeypatch):
"openviking_cli.utils.ollama.detect_ollama_in_config",
lambda config: (False, None, None),
)
- monkeypatch.setattr(
- "openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance",
- lambda: SimpleNamespace(embedding=SimpleNamespace(get_embedder=lambda: None)),
- )
app = create_app(config=ServerConfig(), service=service)
transport = httpx.ASGITransport(app=app)
diff --git a/tests/service/test_task_processing_time.py b/tests/service/test_task_processing_time.py
index b3991c60d..698641c6a 100644
--- a/tests/service/test_task_processing_time.py
+++ b/tests/service/test_task_processing_time.py
@@ -128,9 +128,7 @@ async def test_downstream_work_wait_remains_worker_occupied_time(monkeypatch):
await tracker.register_work("task", "embedding", "Embedding", **OWNER)
now[0] = 3
- waiter = asyncio.create_task(
- tracker.wait_for_work("task", ["embedding"], poll_interval=0.001, **OWNER)
- )
+ waiter = asyncio.create_task(tracker.wait_for_work("task", ["embedding"], **OWNER))
await asyncio.sleep(0.01)
now[0] = 20
assert (await tracker.get("task", **OWNER)).processing_seconds == 20
diff --git a/tests/service/test_task_work_contract.py b/tests/service/test_task_work_contract.py
index 1c302138d..0c4b74b9b 100644
--- a/tests/service/test_task_work_contract.py
+++ b/tests/service/test_task_work_contract.py
@@ -6,17 +6,220 @@ import asyncio
import json
import os
from pathlib import Path
+from typing import Any
from unittest.mock import Mock, create_autospec
from uuid import uuid4
import pytest
-from openviking.service.task_domain import TaskRetentionPolicy, TaskStatus, WorkState
+from openviking.service.task_domain import TaskPatch, TaskRetentionPolicy, TaskStatus, WorkState
from openviking.service.task_store import PersistentTaskStore, WorkRegistration
from openviking.service.task_tracker import TaskTracker
+from openviking_cli.exceptions import FailedPreconditionError
from tests.test_task_tracker import _FakeAgfs
-OWNER = {"account_id": "account", "user_id": "user"}
+OWNER: dict[str, Any] = {"account_id": "account", "user_id": "user"}
+
+
[email protected](params=["file", "mysql"])
+async def task_stores(request):
+ if request.param == "file":
+ store = PersistentTaskStore(_FakeAgfs())
+ yield store, store
+ return
+ raw = os.environ.get("OV_TEST_MYSQL")
+ if not raw:
+ pytest.skip("Set OV_TEST_MYSQL to exercise the shared Task/Work contract")
+ sqlalchemy = pytest.importorskip("sqlalchemy.ext.asyncio")
+ pytest.importorskip("aiomysql")
+ from openviking.service import task_store_mysql
+
+ params = {**json.loads(raw), "table": f"ov_contract_{uuid4().hex[:12]}"}
+ first = task_store_mysql.MySQLTaskWorkStore(params=params)
+ second = task_store_mysql.MySQLTaskWorkStore(params=params)
+ admin = sqlalchemy.create_async_engine(first._url, **first._engine_options)
+ sql_dir = Path(task_store_mysql.__file__).parent / "sql"
+ schema_files = [
+ sql_dir / "task_store_init.sql",
+ sql_dir / "changes/0001_task_store_service_name.sql",
+ ]
+ try:
+ async with admin.begin() as connection:
+ for schema in schema_files:
+ for statement in schema.read_text().replace("ov_tasks", params["table"]).split(";"):
+ if statement.strip():
+ await connection.exec_driver_sql(statement)
+ await first.load()
+ await second.load()
+ yield first, second
+ finally:
+ await first.close()
+ await second.close()
+ async with admin.begin() as connection:
+ await connection.run_sync(first._models.Base.metadata.drop_all)
+ await admin.dispose()
+
+
+async def test_concurrent_events_preserve_history_and_lifecycle(task_stores):
+ first, second = task_stores
+ writer, observer = TaskTracker(first), TaskTracker(second)
+ await writer.create("content_write", task_id="task", **OWNER)
+ await writer.start("task", stage="parsing", **OWNER)
+ initial = await observer.get_with_events("task", **OWNER)
+ assert initial is not None and initial.execution_events is not None
+ before = initial.execution_events
+ await asyncio.gather(
+ *[
+ tracker.record_event(
+ "task", "waiting_for_descendants", operation=f"work-{index}", **OWNER
+ )
+ for index, tracker in enumerate([writer, observer] * 6)
+ ]
+ )
+ await writer.start("task", stage="parsing", **OWNER)
+ await observer.patch_task("task", TaskPatch(stage="embedding"), **OWNER)
+ await writer.fail("task", "provider rejected sk-testsecret123", **OWNER)
+ task = await observer.get_with_events("task", **OWNER)
+ assert task is not None and task.execution_events is not None
+ history = task.execution_events
+ items = history["items"]
+ assert items[: len(before["items"])] == before["items"]
+ assert [item["seq"] for item in items] == list(range(1, 19))
+ assert {item["operation"] for item in items[3:15]} == {f"work-{index}" for index in range(12)}
+ assert [item["kind"] for item in items[15:]] == [
+ "stage_changed",
+ "error_recorded",
+ "status_changed",
+ ]
+ assert items[-1]["stage"] == "embedding"
+ assert items[-1]["status"] == "failed"
+ assert "[REDACTED]" in items[-2]["error"]
+ assert "testsecret123" not in json.dumps(history)
+ assert history["dropped_count"] == 0
+ assert history["started_mid_task"] is False
+ assert task.updated_at > task.created_at
+
+
[email protected]("stage_prefix", ["stage", "阶段" * 64])
+async def test_shared_event_history_retains_latest_metadata_after_truncation(
+ task_stores, stage_prefix
+):
+ from openviking.service.task_events import MAX_TASK_EVENT_BYTES, MAX_TASK_EVENTS
+
+ first, second = task_stores
+ writer, observer = TaskTracker(first), TaskTracker(second)
+ await writer.create("add_resource", task_id="task", **OWNER)
+ for index in range(80):
+ await writer.patch_task("task", TaskPatch(stage=f"{index}:{stage_prefix}"), **OWNER)
+ await writer.cancel("task", **OWNER)
+ task = await observer.get_with_events("task", **OWNER)
+ assert task is not None and task.execution_events is not None
+ history = task.execution_events
+ items = history["items"]
+ assert len(items) <= MAX_TASK_EVENTS
+ assert len(json.dumps(history).encode()) <= MAX_TASK_EVENT_BYTES
+ assert history["dropped_count"] + len(items) == 83
+ assert [item["seq"] for item in items] == list(range(history["dropped_count"] + 1, 84))
+ assert [item["status"] for item in items[-2:]] == ["cancelling", "cancelled"]
+ assert history["started_mid_task"] is False
+
+
+async def test_store_withdraws_only_tasks_without_registered_work(task_stores):
+ first, second = task_stores
+ tracker = TaskTracker(first)
+ await tracker.create("compile", task_id="unused", **OWNER)
+ assert (await second.withdraw_unaccepted("unused", **OWNER)).is_applied
+ assert await tracker.get("unused", **OWNER) is None
+
+ await tracker.create("compile", task_id="accepted", **OWNER)
+ assert (
+ await tracker.register_work("accepted", "work", "ExternalTask", **OWNER)
+ is WorkRegistration.CREATED
+ )
+ assert not (await second.withdraw_unaccepted("accepted", **OWNER)).accepted
+ assert await tracker.get("accepted", **OWNER) is not None
+ assert await tracker.get_work_state("accepted", "work", **OWNER) is WorkState.PENDING
+
+
+async def test_withdrawal_is_atomic_with_work_registration(task_stores):
+ first, second = task_stores
+ tracker = TaskTracker(first)
+ await tracker.create("compile", task_id="task", **OWNER)
+
+ withdrawn, registered = await asyncio.gather(
+ second.withdraw_unaccepted("task", **OWNER),
+ tracker.register_work("task", "work", "ExternalTask", **OWNER),
+ )
+ if withdrawn.is_applied:
+ assert registered is WorkRegistration.REJECTED
+ assert await tracker.get("task", **OWNER) is None
+ else:
+ assert not withdrawn.accepted
+ assert registered is WorkRegistration.CREATED
+ assert await tracker.get_work_state("task", "work", **OWNER) is WorkState.PENDING
+
+
[email protected]("postcheck", ["query-error", "refused"])
+async def test_creator_postcheck_preserves_or_cancels_accepted_task(monkeypatch, postcheck):
+ store = PersistentTaskStore(_FakeAgfs())
+ creator, replay = TaskTracker(store), TaskTracker(store)
+ inserted, release = asyncio.Event(), asyncio.Event()
+ create = store.create
+ checks = 0
+ query_error = ConnectionError("Identity storage unavailable")
+
+ async def guard(account_id, user_id):
+ nonlocal checks
+ checks += 1
+ if checks == 2:
+ if postcheck == "query-error":
+ raise query_error
+ return True
+ return False
+
+ async def delayed_create(task, **kwargs):
+ result = await create(task, **kwargs)
+ inserted.set()
+ await release.wait()
+ return result
+
+ creator.set_deletion_guard(guard)
+ monkeypatch.setattr(store, "create", delayed_create)
+ request = asyncio.create_task(creator.create("compile", task_id="task", **OWNER))
+ try:
+ await asyncio.wait_for(inserted.wait(), 1)
+ await replay.create("compile", task_id="task", **OWNER)
+ assert (
+ await replay.register_work("task", "work", "ExternalTask", **OWNER)
+ is WorkRegistration.CREATED
+ )
+ release.set()
+ with pytest.raises(
+ ConnectionError if postcheck == "query-error" else FailedPreconditionError
+ ) as raised:
+ await asyncio.wait_for(request, 1)
+ task = await replay.get("task", **OWNER)
+ assert task is not None
+ assert await replay.get_work_state("task", "work", **OWNER) is WorkState.PENDING
+
+ if postcheck == "query-error":
+ assert raised.value is query_error
+ assert task.status is TaskStatus.PENDING
+ await replay.cancel("task", **OWNER)
+ else:
+ assert task.status is TaskStatus.CANCELLING
+
+ await replay.mark_work_done("task", "work", **OWNER)
+ assert (await replay.wait("task", timeout=0, **OWNER)).status is TaskStatus.CANCELLED
+ finally:
+ release.set()
+ request.cancel()
+ await asyncio.gather(request, return_exceptions=True)
@pytest.fixture
@@ -43,7 +246,7 @@ async def test_task_work_wait_includes_recovered_and_later_enqueued_work(tracker
await tracker.register_work("task", "root", "Inline", **OWNER)
await tracker.reconcile_delivery("task", "semantic", "Semantic", **OWNER)
waiting = asyncio.create_task(
- tracker.wait_for_task_work("task", exclude_work_id="root", poll_interval=0.001, **OWNER)
+ tracker.wait_for_task_work("task", exclude_work_id="root", **OWNER)
)
try:
await asyncio.sleep(0)
@@ -199,6 +402,15 @@ async def test_mysql_requeue_and_concurrent_completion_are_durable(deletion):
assert isolated_task is not None
await isolated.register_work(task_id, "semantic", "Semantic", **OWNER)
+ assert (await second.withdraw_unaccepted(task_id, **OWNER)).is_applied
+ assert await observer.get(task_id, **OWNER) is None
+ assert await isolated.get(task_id, **OWNER) is not None
+ assert await isolated.get_work_state(task_id, "semantic", **OWNER) is WorkState.PENDING
+ assert (
+ await writer.create_if_no_running("content_write", uri, task_id=task_id, **OWNER)
+ is not None
+ )
+
await writer.start(task_id, **OWNER)
await writer.register_work(task_id, "semantic", "Semantic", **OWNER)
await observer.register_work(task_id, "embedding", "Embedding", **OWNER)
diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.py
index 34d1df5e5..1d724a4ce 100644
--- a/tests/test_task_tracker.py
+++ b/tests/test_task_tracker.py
@@ -17,6 +17,7 @@ from openviking.server.identity import RequestContext, Role
from openviking.service.session_service import SessionService
from openviking.service.task_domain import TaskPatch
from openviking.service.task_store import (
+ SYSTEM_TASK_ACCOUNT_ID,
PersistentTaskStore,
_task_from_payload,
_task_to_payload,
@@ -28,9 +29,10 @@ from openviking.service.task_tracker import (
get_task_tracker,
run_tracked_operation,
set_task_tracker,
+ task_admission,
)
from openviking.service.task_work_index import get_task_context
-from openviking_cli.exceptions import DeadlineExceededError
+from openviking_cli.exceptions import DeadlineExceededError, FailedPreconditionError
from openviking_cli.session.user_id import UserIdentifier
pytestmark = pytest.mark.asyncio
@@ -177,6 +179,178 @@ async def test_create_task(tracker: TaskTracker):
assert task.status == TaskStatus.PENDING
+async def test_deletion_guard_refuses_new_tasks_for_deleting_identities(tracker: TaskTracker):
+ existing = await tracker.create("add_resource", task_id="replayed", **_owner_kwargs())
+ deleting = {("acme", "alice")}
+
+ async def guard(account_id, user_id):
+ return (account_id, user_id) in deleting
+
+ tracker.set_deletion_guard(guard)
+
+ with pytest.raises(FailedPreconditionError):
+ await tracker.create("add_resource", task_id="late", **_owner_kwargs())
+ assert await tracker.get("late", **_owner_kwargs()) is None
+ replayed = await tracker.create("add_resource", task_id="replayed", **_owner_kwargs())
+ assert replayed.task_id == existing.task_id
+ assert await tracker.create("add_resource", **_owner_kwargs(user_id="bob"))
+ assert await tracker.create("account_delete", account_id=SYSTEM_TASK_ACCOUNT_ID, user_id="root")
+
+
+async def test_identity_cleanup_task_creation_allows_deleting_owner(tracker: TaskTracker):
+ async def guard(account_id, user_id):
+ return (account_id, user_id) == ("acc", "alice")
+
+ tracker.set_deletion_guard(guard)
+
+ task = await tracker.create(
+ "user_delete",
+ task_id="cleanup",
+ account_id="acc",
+ user_id="alice",
+ )
+ assert task.task_id == "cleanup"
+ with pytest.raises(FailedPreconditionError):
+ await tracker.create("add_resource", account_id="acc", user_id="alice")
+
+
+async def test_wait_uses_short_initial_poll(monkeypatch, tracker: TaskTracker):
+ task = await tracker.create("add_resource", task_id="task", **_owner_kwargs())
+ delays = []
+
+ async def sleep(delay):
+ delays.append(delay)
+ await tracker.complete(task.task_id, {}, **_owner_kwargs())
+
+ monkeypatch.setattr(asyncio, "sleep", sleep)
+
+ assert (await tracker.wait(task.task_id, **_owner_kwargs())).status is TaskStatus.COMPLETED
+ assert delays == [0.05]
+
+
+async def test_wait_for_task_work_uses_short_initial_poll(monkeypatch, tracker: TaskTracker):
+ task = await tracker.create("add_resource", task_id="task", **_owner_kwargs())
+ await tracker.start(task.task_id, **_owner_kwargs())
+ await tracker.register_work(task.task_id, "work", "Semantic", **_owner_kwargs())
+ delays = []
+
+ async def sleep(delay):
+ delays.append(delay)
+ await tracker.mark_work_done(task.task_id, "work", **_owner_kwargs())
+
+ monkeypatch.setattr(asyncio, "sleep", sleep)
+
+ await tracker.wait_for_task_work(task.task_id, **_owner_kwargs())
+ assert delays == [0.05]
+
+
[email protected]("failure", ["deletion", "identity-query", "watch-admission"])
+async def test_creation_rejected_after_insert_withdraws_the_task(tracker, failure):
+ deleting = set()
+ store_create = tracker._store.create
+ error = ConnectionError("Admission storage unavailable")
+
+ async def guard(account_id, user_id):
+ if (account_id, user_id) not in deleting:
+ return False
+ if failure == "identity-query":
+ raise error
+ return failure == "deletion"
+
+ async def admitted():
+ if deleting and failure == "watch-admission":
+ raise error
+ return True
+
+ async def create_after_cleanup(task, **kwargs):
+ # The check has passed; deletion fences the identity and finishes
+ # listing Tasks before this insert lands.
+ deleting.add((task.account_id, task.user_id))
+ assert await tracker.list_tasks(**_owner_kwargs(), limit=None) == []
+ return await store_create(task, **kwargs)
+
+ tracker.set_deletion_guard(guard)
+ tracker._store.create = create_after_cleanup
+
+ with task_admission("late", admitted):
+ with pytest.raises(
+ FailedPreconditionError if failure == "deletion" else ConnectionError
+ ) as raised:
+ await tracker.create(
+ "connector_import", resource_id="resource", task_id="late", **_owner_kwargs()
+ )
+ if failure != "deletion":
+ assert raised.value is error
+ assert await tracker.get("late", **_owner_kwargs()) is None
+ tracker._store.create = store_create
+ deleting.clear()
+ assert await tracker.create_if_no_running("connector_import", "resource", **_owner_kwargs())
+
+
[email protected]("cancel_at", ["insert", "identity-query", "watch-admission"])
+async def test_cancel_during_task_creation_withdraws_the_unaccepted_task(cancel_at):
+ entered = asyncio.Event()
+ release = threading.Event()
+ loop = asyncio.get_running_loop()
+
+ class BlockingAgfs(_FakeAgfs):
+ def write(self, path, data, *args, **kwargs):
+ if cancel_at == "insert" and json.loads(data)["status"] == "pending":
+ loop.call_soon_threadsafe(entered.set)
+ assert release.wait(3)
+ return super().write(path, data, *args, **kwargs)
+
+ agfs = BlockingAgfs()
+ tracker = TaskTracker(PersistentTaskStore(agfs))
+ checks = 0
+
+ async def check():
+ nonlocal checks
+ checks += 1
+ if checks == 2:
+ entered.set()
+ await asyncio.Event().wait()
+
+ async def guard(account_id, user_id):
+ if cancel_at == "identity-query":
+ await check()
+ return False
+
+ async def admitted():
+ if cancel_at == "watch-admission":
+ await check()
+ return True
+
+ tracker.set_deletion_guard(guard)
+ with task_admission("late", admitted):
+ running = asyncio.create_task(
+ tracker.create("connector_import", task_id="late", **_owner_kwargs())
+ )
+ try:
+ await asyncio.wait_for(entered.wait(), 1)
+ running.cancel()
+ await asyncio.sleep(0)
+ running.cancel()
+ finally:
+ release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(running, 1)
+
+ assert await TaskTracker(PersistentTaskStore(agfs)).list_tasks(**_owner_kwargs()) == []
+
+
+async def test_admission_gates_only_its_own_task_id(tracker: TaskTracker):
+ async def refused():
+ return False
+
+ with task_admission("run-1", refused):
+ with pytest.raises(FailedPreconditionError):
+ await tracker.create("add_resource", task_id="run-1", **_owner_kwargs())
+ assert await tracker.create("add_resource", task_id="other", **_owner_kwargs())
+ assert await tracker.get("run-1", **_owner_kwargs()) is None
+ assert await tracker.create("add_resource", task_id="run-1", **_owner_kwargs())
+
+
async def test_start_task(tracker: TaskTracker):
task = await tracker.create("session_commit", **_owner_kwargs())
await tracker.start(task.task_id)
diff --git a/tests/test_task_tracker_concurrency.py b/tests/test_task_tracker_concurrency.py
index 941f91b8e..f4a5253e3 100644
--- a/tests/test_task_tracker_concurrency.py
+++ b/tests/test_task_tracker_concurrency.py
@@ -265,29 +265,56 @@ async def test_keyed_lock_registry_is_cleaned_after_waiter_cancellation():
@pytest.mark.asyncio
-async def test_file_store_concurrent_fixed_id_create_is_idempotent():
[email protected]("failure", ["none", "identity-query", "cancel"])
+async def test_file_store_concurrent_fixed_id_create_is_idempotent(failure):
from openviking.service.task_store import PersistentTaskStore
- from openviking.service.task_tracker import TaskTracker
+ from openviking.service.task_tracker import TaskStatus, TaskTracker
from tests.test_task_tracker import _FakeAgfs
- tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))
- first, second = await asyncio.gather(
- tracker.create(
- "add_resource",
- task_id="fixed",
- account_id="acme",
- user_id="alice",
- ),
- tracker.create(
- "add_resource",
- task_id="fixed",
- account_id="acme",
- user_id="alice",
- ),
+ store = PersistentTaskStore(_FakeAgfs())
+ writer, replay = TaskTracker(store), TaskTracker(store)
+ entered, release = asyncio.Event(), asyncio.Event()
+ create = store.create
+
+ async def delayed_create(task, **kwargs):
+ if task.meta.get("delayed"):
+ entered.set()
+ await release.wait()
+ return await create(task, **kwargs)
+
+ store.create = delayed_create
+ checks = 0
+
+ async def guard(account_id, user_id):
+ nonlocal checks
+ checks += 1
+ if checks > 1 and failure == "identity-query":
+ raise ConnectionError("Identity storage unavailable")
+ return False
+
+ replay.set_deletion_guard(guard)
+ owner = {"account_id": "acme", "user_id": "alice"}
+ duplicate = asyncio.create_task(
+ replay.create("add_resource", task_id="fixed", meta={"delayed": True}, **owner)
)
-
- assert first.task_id == second.task_id == "fixed"
- assert len(await tracker.list_tasks(account_id="acme", user_id="alice")) == 1
+ try:
+ await asyncio.wait_for(entered.wait(), 1)
+ first = await writer.create("add_resource", task_id="fixed", **owner)
+ await writer.start(first.task_id, **owner)
+ await writer.register_work(first.task_id, "work", "Semantic", **owner)
+ if failure == "cancel":
+ duplicate.cancel()
+ finally:
+ release.set()
+ if failure == "cancel":
+ with pytest.raises(asyncio.CancelledError):
+ await duplicate
+ else:
+ second = await duplicate
+ assert first.task_id == second.task_id == "fixed"
+ assert (await writer.get("fixed", **owner)).status is TaskStatus.RUNNING
+ assert await writer.has_work("fixed", **owner)
+ assert len(await writer.list_tasks(**owner)) == 1
@pytest.mark.asyncio