Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(tasks): fence admission and preserve accepted task ownership (#6723)
runyunzhou2e8e53b28d8f
diff --git a/openviking/server/account_stores/file.py b/openviking/server/account_stores/file.pyindex 01b5c6788..16c2853f6 100644--- a/openviking/server/account_stores/file.py+++ b/openviking/server/account_stores/file.py@@ -15,6 +15,7 @@ from argon2 import PasswordHasherfrom argon2.exceptions import VerificationErrorfrom typing_extensions import deprecated+from openviking.concurrency import AsyncSemaphorefrom openviking.core.identifiers import validate_identifier_lengthfrom openviking.pyagfs import AGFSAlreadyExistsError, AGFSNotFoundError, AsyncAGFSClientfrom openviking.pyagfs.async_client import fs_ctx_from_agfs_path@@ -135,11 +136,11 @@ class FileStore(AccountStore):self._prefix_index: Dict[str, list[UserKeyEntry]] = {}self._user_group_ids: Dict[tuple[str, str], tuple[str, ...]] = {}# Serializes internal refreshes so overlapping reloads can't interleave.- self._reload_lock = asyncio.Lock()+ self._reload_lock = AsyncSemaphore(1)# Serializes all mutations made by the unified manager in this process.# File locks remain responsible for inter-instance coordination.- self._mutation_lock = asyncio.Lock()- self._deletion_lock = asyncio.Lock()+ self._mutation_lock = AsyncSemaphore(1)+ self._deletion_lock = AsyncSemaphore(1)def _discard_account_state(self, account_id: str) -> None:"""Remove an account and its key index entries from in-memory state."""diff --git a/openviking/server/app.py b/openviking/server/app.pyindex 5ee94688f..7809a6540 100644--- a/openviking/server/app.py+++ b/openviking/server/app.py@@ -189,6 +189,7 @@ async def _initialize_runtime_state(oauth_store=getattr(app.state, "oauth_store", None),usage_audit_runtime=getattr(app.state, "usage_audit_runtime", None),gateway_app=getattr(app.state, "gateway_app", None),+ require_registered_identity=config.get_effective_auth_mode() == "api_key",)logger.info("OpenVikingService initialization complete")diff --git a/openviking/service/deletion.py b/openviking/service/deletion.pyindex f6b66253d..ad6d2d696 100644--- a/openviking/service/deletion.py+++ b/openviking/service/deletion.py@@ -71,6 +71,7 @@ class DeletionService:oauth_store: Any = None,usage_audit_runtime: Any = None,gateway_app: Any = None,+ require_registered_identity: bool = True,) -> None:self._service = serviceself._manager = manager@@ -79,11 +80,16 @@ class DeletionService:self._oauth_store = oauth_storeself._usage_audit_runtime = usage_audit_runtimeself._gateway_app = gateway_app+ self._require_registered_identity = require_registered_identityself._request_lock = asyncio.Lock()async def initialize(self) -> None:"""Bind the queue consumer and reconcile persisted deletion fences."""self._service.viking_fs.set_deletion_guard(self._is_deleting)+ creation_guard = (+ self._creation_blocked if self._require_registered_identity else self._is_deleting+ )+ get_task_tracker().set_deletion_guard(creation_guard)queue_manager = self._service._queue_managerqueue = queue_manager.get_queue(queue_manager.DATA_CLEANUP)queued_task_ids = {@@ -134,6 +140,15 @@ class DeletionService:)return deletion is not None+ async def _creation_blocked(self, account_id: str, user_id: str) -> bool:+ """Reject missing registered identities, also after their deletion fence is gone."""+ if account_id == SYSTEM_TASK_ACCOUNT_ID:+ return False+ if await self._is_deleting(account_id, user_id):+ return True+ user = await self._manager.get_user(account_id, user_id)+ return user is None+async def delete(self,account_id: str,@@ -420,9 +435,16 @@ class DeletionService:remaining = deadline - time.monotonic()if remaining <= 0:raise TimeoutError("Timed out waiting for tasks to stop")- await tracker.wait(- task.task_id, account_id=task.account_id, user_id=task.user_id, timeout=remaining- )+ try:+ await tracker.wait(+ task.task_id,+ account_id=task.account_id,+ user_id=task.user_id,+ timeout=remaining,+ )+ except KeyError:+ # A Task created concurrently with this deletion withdraws itself.+ continueasync def _delete_uploads(self, ctx: RequestContext) -> None:viking_fs = self._service.viking_fs@@ -513,6 +535,7 @@ async def setup_deletion(oauth_store: Any = None,usage_audit_runtime: Any = None,gateway_app: Any = None,+ require_registered_identity: bool = True,) -> Optional[DeletionService]:"""Create the deletion service after storage and authentication are ready."""if manager is None or service.viking_fs is None or service._queue_manager is None:@@ -524,6 +547,7 @@ async def setup_deletion(oauth_store=oauth_store,usage_audit_runtime=usage_audit_runtime,gateway_app=gateway_app,+ require_registered_identity=require_registered_identity,)await deletion_service.initialize()return deletion_servicediff --git a/openviking/service/resource_memory_link_service.py b/openviking/service/resource_memory_link_service.pyindex 04a029645..cefc906a8 100644--- a/openviking/service/resource_memory_link_service.py+++ b/openviking/service/resource_memory_link_service.py@@ -338,7 +338,6 @@ class ResourceMemoryLinkService:account_id=ctx.account_id,user_id=ctx.user.user_id,timeout=_RESOURCE_REASON_COMMIT_TIMEOUT_SECONDS,- poll_interval=0.1,)if task.status == TaskStatus.COMPLETED:return task.to_dict()diff --git a/openviking/service/task_store.py b/openviking/service/task_store.pyindex 7924cbb43..ad852acd3 100644--- a/openviking/service/task_store.py+++ b/openviking/service/task_store.py@@ -112,6 +112,14 @@ class StoreOperationResult:return cls(StoreOperationOutcome.REJECTED, code, message)+@dataclass(frozen=True)+class TaskCreationResult:+ """The stored Task and whether this call inserted it."""++ task: TaskRecord+ created: bool++@dataclass(frozen=True)class TaskDeletionBlocker:task_id: str@@ -150,7 +158,19 @@ class TaskWorkStore(Protocol):@propertydef work_recovery_mode(self) -> WorkRecoveryMode: ...- async def create(self, task: TaskRecord, *, reject_if_active: bool = False) -> TaskRecord: ...+ async def create(+ self, task: TaskRecord, *, reject_if_active: bool = False+ ) -> TaskCreationResult: ...++ async def withdraw_unaccepted(+ self, task_id: str, *, account_id: str, user_id: str+ ) -> StoreOperationResult:+ """Atomically delete an initial pending Task only while it has no Work.++ Reject Tasks with execution progress or any retained Work, including+ closed Work. Serialize this decision with Task and Work mutations.+ """+ ...async def get_task(self,@@ -352,6 +372,17 @@ class TaskWorkStore(Protocol):TaskStore = TaskWorkStore+def _can_withdraw_task(task: TaskRecord, *, has_work: bool) -> bool:+ return (+ task.status is TaskStatus.PENDING+ and not has_work+ and task.stage is None+ and task.result is None+ and task.error is None+ and task.processing_seconds is None+ )++@dataclassclass _WorkSet:works: Dict[str, WorkRecord]@@ -392,7 +423,9 @@ class PersistentTaskStore(TaskWorkStore):def work_recovery_mode(self) -> WorkRecoveryMode:return WorkRecoveryMode.QUEUE_SNAPSHOT- async def create(self, task: TaskRecord, *, reject_if_active: bool = False) -> TaskRecord:+ async def create(+ self, task: TaskRecord, *, reject_if_active: bool = False+ ) -> TaskCreationResult:if not task.account_id or not task.user_id:raise ValueError("PersistentTaskStore requires account_id and user_id")business_key = (task.account_id, task.user_id, task.task_type, task.resource_id)@@ -410,14 +443,34 @@ class PersistentTaskStore(TaskWorkStore):async with self._record_locks.acquire(owner):existing = await self._load_task(task.task_id, task.account_id, task.user_id)if existing is not None:- return self._copy_task(existing, self._work_sets.get(owner))+ return TaskCreationResult(+ self._copy_task(existing, self._work_sets.get(owner)), created=False+ )stored = deepcopy(task)append_task_events(stored, None)await self._write_task(stored)with self._lock:self._task_cache[owner] = storedself._work_sets[owner] = _WorkSet({})- return deepcopy(stored)+ return TaskCreationResult(deepcopy(stored), created=True)++ async def withdraw_unaccepted(+ self, task_id: str, *, account_id: str, user_id: str+ ) -> StoreOperationResult:+ if not account_id or not user_id:+ raise ValueError("Task withdrawal requires account_id and user_id")+ owner = (account_id, user_id, task_id)+ async with self._record_locks.acquire(owner):+ task = await self._load_task(task_id, account_id, user_id)+ if task is None:+ return StoreOperationResult.noop(StoreOperationCode.NOT_FOUND)+ work_set = self._work_sets.get(owner)+ if not _can_withdraw_task(task, has_work=bool(work_set and work_set.works)):+ return StoreOperationResult.rejected(+ StoreOperationCode.INVALID_STATE, "Task has already been used"+ )+ await self._delete_task_unlocked(owner, task_exists=True)+ return StoreOperationResult.applied()async def get_task(self,diff --git a/openviking/service/task_store_mysql.py b/openviking/service/task_store_mysql.pyindex 9db121819..b3a04542f 100644--- a/openviking/service/task_store_mysql.py+++ b/openviking/service/task_store_mysql.py@@ -42,15 +42,17 @@ from openviking.service.task_domain import (sanitize_error,summarize_works,)-from openviking.service.task_events import append_task_event+from openviking.service.task_events import TaskEventHistory, append_task_eventfrom openviking.service.task_store import (DeleteTasksResult,StoreOperationCode,StoreOperationResult,TaskAlreadyActive,+ TaskCreationResult,TaskDeletionBlocker,TaskStatsSnapshot,TaskWorkStore,+ _can_withdraw_task,_deep_merge,)from openviking.service.task_store_mysql_schema import (@@ -299,7 +301,7 @@ class MySQLTaskWorkStore(TaskWorkStore):def _task_from_row(row: Any,works: Iterable[WorkRecord] = (),- execution_events: Optional[dict] = None,+ execution_events: Optional[TaskEventHistory] = None,) -> TaskRecord:task = TaskRecord(task_id=row.task_id,@@ -418,7 +420,9 @@ class MySQLTaskWorkStore(TaskWorkStore):rows = (await session.scalars(statement)).all()return [self._work_from_row(row) for row in rows]- async def _event_history(self, session: AsyncSession, task_id: str) -> Optional[dict]:+ async def _event_history(+ self, session: AsyncSession, task_id: str+ ) -> Optional[TaskEventHistory]:Event = self._models.Eventrows = (await session.scalars(@@ -445,36 +449,27 @@ class MySQLTaskWorkStore(TaskWorkStore):"started_mid_task": bool(latest.started_mid_task),}- async def _store_event_history(+ async def _append_event_history(self,session: AsyncSession,task_id: str,- history: Optional[dict],+ history: Optional[TaskEventHistory],+ previous: Optional[TaskEventHistory] = None,) -> None:+ """Insert new events under the Task lock; retained rows remain immutable."""if history is None:returnEvent = self._models.Eventitems = history["items"]if not items:return- seqs = [int(item["seq"]) for item in items]- existing = set(- await session.scalars(- select(Event.seq).where(- *self._event_query(Event, task_id),- Event.seq.in_(seqs),- )- )- )+ previous_seq = previous["items"][-1]["seq"] if previous and previous["items"] else 0metadata = {"dropped_count": int(history["dropped_count"]),"started_mid_task": bool(history["started_mid_task"]),}- await session.execute(- update(Event).where(*self._event_query(Event, task_id)).values(**metadata)- )for item in items:- if int(item["seq"]) in existing:+ if item["seq"] <= previous_seq:continuesession.add(Event(@@ -490,9 +485,10 @@ class MySQLTaskWorkStore(TaskWorkStore):**metadata,))- await session.execute(- delete(Event).where(*self._event_query(Event, task_id), Event.seq < min(seqs))- )+ if history["dropped_count"] > (previous["dropped_count"] if previous else 0):+ await session.execute(+ delete(Event).where(*self._event_query(Event, task_id), Event.seq < items[0]["seq"])+ )async def _lock_work(self, session: AsyncSession, task_id: str, work_id: str) -> Any:Work = self._models.Work@@ -508,7 +504,9 @@ class MySQLTaskWorkStore(TaskWorkStore):async with session.begin():yield session- async def create(self, task: TaskRecord, *, reject_if_active: bool = False) -> TaskRecord:+ async def create(+ self, task: TaskRecord, *, reject_if_active: bool = False+ ) -> TaskCreationResult:account_id, user_id = self._require_owner(task.account_id, task.user_id)Task = self._models.Tasknow = time.time()@@ -560,12 +558,15 @@ class MySQLTaskWorkStore(TaskWorkStore):raise ValueError(f"Task id already belongs to another owner: {stored.task_id}")- return self._task_from_row(- existing,- execution_events=await self._event_history(session, stored.task_id),+ return TaskCreationResult(+ self._task_from_row(+ existing,+ execution_events=await self._event_history(session, stored.task_id),+ ),+ created=False,)session.add(row)- await self._store_event_history(session, stored.task_id, stored.execution_events)+ await self._append_event_history(session, stored.task_id, stored.execution_events)except IntegrityError as exc:if not self._is_duplicate_error(exc):raise@@ -581,14 +582,17 @@ class MySQLTaskWorkStore(TaskWorkStore):raise ValueError(f"Task id already belongs to another owner: {stored.task_id}")- return self._task_from_row(- existing,- execution_events=await self._event_history(session, stored.task_id),+ return TaskCreationResult(+ self._task_from_row(+ existing,+ execution_events=await self._event_history(session, stored.task_id),+ ),+ created=False,)if reject_if_active:raise TaskAlreadyActive(stored.task_type, stored.resource_id) from excraise- return stored+ return TaskCreationResult(stored, created=True)@staticmethoddef _is_duplicate_error(exc: IntegrityError) -> bool:@@ -776,6 +780,34 @@ class MySQLTaskWorkStore(TaskWorkStore):))+ async def withdraw_unaccepted(+ self, task_id: str, *, account_id: str, user_id: str+ ) -> StoreOperationResult:+ self._require_owner(account_id, user_id)+ Work = self._models.Work+ async with self._transaction() as session:+ row = await self._load_task(+ session, task_id, account_id=account_id, user_id=user_id, lock=True+ )+ if row is None:+ return StoreOperationResult.noop(StoreOperationCode.NOT_FOUND)+ has_work = bool(+ await session.scalar(+ select(+ exists().where(+ Work.service_name == self._service_name,+ Work.task_id == task_id,+ )+ )+ )+ )+ if not _can_withdraw_task(self._task_from_row(row), has_work=has_work):+ return StoreOperationResult.rejected(+ StoreOperationCode.INVALID_STATE, "Task has already been used"+ )+ await self._delete_task_records(session, [task_id])+ return StoreOperationResult.applied()+async def delete_tasks(self,task_id: Optional[str] = None,@@ -969,7 +1001,9 @@ class MySQLTaskWorkStore(TaskWorkStore):active_business_key=None,))- await self._store_event_history(session, task_id, task.execution_events)+ await self._append_event_history(+ session, task_id, task.execution_events, previous.execution_events+ )return taskasync def start_task(@@ -1014,7 +1048,9 @@ class MySQLTaskWorkStore(TaskWorkStore):updated_at=TaskRecord.next_updated_at(row.updated_at),))- await self._store_event_history(session, task_id, task.execution_events)+ await self._append_event_history(+ session, task_id, task.execution_events, previous.execution_events+ )return StoreOperationResult.applied()async def patch_task(@@ -1070,7 +1106,9 @@ class MySQLTaskWorkStore(TaskWorkStore):task = deepcopy(previous)task.stage = stageappend_task_events(task, previous)- await self._store_event_history(session, task_id, task.execution_events)+ await self._append_event_history(+ session, task_id, task.execution_events, previous.execution_events+ )if "meta" in patch.clear_fields or patch.meta is not None:meta = {}if "meta" not in patch.clear_fields:@@ -1126,8 +1164,9 @@ class MySQLTaskWorkStore(TaskWorkStore):StoreOperationCode.ALREADY_TERMINAL,f"Task is already {row.status}",)+ previous = await self._event_history(session, task_id)events = append_task_event(- await self._event_history(session, task_id),+ deepcopy(previous),kind=kind,status=row.status,stage=sanitize_error(row.stage) if row.stage else None,@@ -1140,7 +1179,7 @@ class MySQLTaskWorkStore(TaskWorkStore):updated_at=TaskRecord.next_updated_at(row.updated_at),))- await self._store_event_history(session, task_id, events)+ await self._append_event_history(session, task_id, events, previous)return StoreOperationResult.applied()async def record_outcome(@@ -1204,6 +1243,7 @@ class MySQLTaskWorkStore(TaskWorkStore):next_status = TaskStatus.CANCELLING if cancelled else TaskStatus(row.status)events = None+ previous_events = Noneif cancelled or (error and row.error is None):previous = TaskRecord(task_id=task_id,@@ -1213,6 +1253,7 @@ class MySQLTaskWorkStore(TaskWorkStore):error=row.error,execution_events=await self._event_history(session, task_id),)+ previous_events = previous.execution_eventstask = deepcopy(previous)task.status = next_statustask.error = next_error@@ -1236,7 +1277,7 @@ class MySQLTaskWorkStore(TaskWorkStore):updated_at=TaskRecord.next_updated_at(row.updated_at),))- await self._store_event_history(session, task_id, events)+ await self._append_event_history(session, task_id, events, previous_events)finalized = await self._finalize_if_closed(session,task_id,@@ -1321,7 +1362,9 @@ class MySQLTaskWorkStore(TaskWorkStore):updated_at=TaskRecord.next_updated_at(row.updated_at),))- await self._store_event_history(session, task_id, task.execution_events)+ await self._append_event_history(+ session, task_id, task.execution_events, previous.execution_events+ )finalized = await self._finalize_if_closed(session,task_id,diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.pyindex 5a8c3e694..ccd442904 100644--- a/openviking/service/task_tracker.py+++ b/openviking/service/task_tracker.py@@ -54,12 +54,17 @@ from openviking.service.task_store import (StoreOperationCode,StoreOperationResult,TaskAlreadyActive,+ TaskCreationResult,TaskWorkStore,WorkRegistration,)from openviking.service.task_tracker_concurrency import run_to_completionfrom openviking.service.task_work_index import TaskWorkIndex-from openviking_cli.exceptions import DeadlineExceededError, OpenVikingError+from openviking_cli.exceptions import (+ DeadlineExceededError,+ FailedPreconditionError,+ OpenVikingError,+)from openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)@@ -85,8 +90,10 @@ _CANCELLABLE_TASK_TYPES = {"admin_reindex","snapshot_restore_reindex",}+_IDENTITY_CLEANUP_TASK_TYPES = {"account_delete", "user_delete"}_CANCEL_POLL_INTERVAL = 30-_TASK_WAIT_POLL_INTERVAL = 0.05+_WAIT_INITIAL_POLL_INTERVAL = 0.05+_WAIT_MAX_POLL_INTERVAL = 10.0_PUBLIC_QUEUE_NAMES = ("Semantic", "Embedding")_sanitize_error = sanitize_error@@ -102,6 +109,28 @@ class _WaitBudget:_wait_budget: ContextVar[Optional[_WaitBudget]] = ContextVar("task_wait_budget", default=None)+_admission: ContextVar[Optional[tuple[str, Callable[[], Awaitable[bool]]]]] = ContextVar(+ "task_admission", default=None+)+++@contextmanager+def task_admission(+ task_id: str, admitted: Callable[[], Awaitable[bool]]+) -> Generator[None, None, None]:+ """Create *task_id* within this context only while *admitted* holds.++ create checks before the insert and again after it. A confirmed revocation+ withdraws an unused Task or cancels one that another caller has taken over.+ A failed check or cancelled caller only withdraws an unused Task.+ """+ token = _admission.set((task_id, admitted))+ try:+ yield+ finally:+ _admission.reset(token)++@contextmanagerdef _consume_wait_budget(task_id: str, timeout: Optional[float]@@ -296,12 +325,52 @@ class TaskTracker:self._running: ContextVar[tuple[str, str, str, bool] | None] = ContextVar("task_tracker_running_work", default=None)+ self._deletion_guard: Optional[Callable[[str, str], Any]] = None@staticmethoddef _validate_owner(account_id: str, user_id: str) -> None:if not account_id or not user_id:raise ValueError("Task ownership requires non-empty account_id and user_id")+ def set_deletion_guard(self, guard: Optional[Callable[[str, str], Any]]) -> None:+ """Reject new Tasks for identities the guard reports as being deleted.++ Identity cleanup records its fence and then lists the Tasks to cancel;+ create inserts the Task and then reads the fence again. One of the two+ reads observes the other's write, so a Task inserted concurrently with+ a deletion is either listed by cleanup or withdrawn/cancelled here.+ """+ self._deletion_guard = guard++ async def _owner_deleting(self, account_id: str, user_id: str) -> bool:+ guard = self._deletion_guard+ if guard is None or account_id == SYSTEM_TASK_ACCOUNT_ID:+ return False+ result = guard(account_id, user_id)+ return bool(await result if inspect.isawaitable(result) else result)++ async def _withdraw(+ self, task_id: str, *, account_id: str, user_id: str, refusal: Optional[str]+ ) -> None:+ withdrawn = await self._store.withdraw_unaccepted(+ task_id, account_id=account_id, user_id=user_id+ )+ if withdrawn.is_applied:+ self._work_index.forget_task(task_id, account_id=account_id, user_id=user_id)+ return+ if withdrawn.accepted or refusal is None:+ return+ cancellation = await self._request_cancellation(+ task_id,+ account_id=account_id,+ user_id=user_id,+ allow_terminal=False,+ )+ if cancellation.accepted:+ self._work_index.cancel_active(task_id, account_id=account_id, user_id=user_id)+ elif cancellation.code is StoreOperationCode.NOT_CANCELLABLE:+ await self.fail(task_id, refusal, account_id=account_id, user_id=user_id)+def attach_work_index(self, work_index: TaskWorkIndex) -> None:"""Replace the process-local active-handle index."""self._work_index = work_index@@ -417,6 +486,15 @@ class TaskTracker:existing = await self.get(task_id, account_id=account_id, user_id=user_id)if existing is not None:return existing+ admission = _admission.get()+ admitted = admission[1] if admission and admission[0] == task_id else None+ # Identity cleanup may be owned by an actor that is itself being deleted;+ # fencing it would strand the target deletion without a cleanup task.+ enforce_identity_admission = task_type not in _IDENTITY_CLEANUP_TASK_TYPES+ if enforce_identity_admission and await self._owner_deleting(account_id, user_id):+ raise FailedPreconditionError("Identity deletion is in progress")+ if admitted is not None and not await admitted():+ raise FailedPreconditionError("Task creation is no longer admitted")task = TaskRecord(task_id=task_id or str(uuid4()),task_type=task_type,@@ -426,10 +504,41 @@ class TaskTracker:meta=dict(meta or {}),auth=dict(auth or {}),)- stored = await run_to_completion(- lambda: self._store.create(task, reject_if_active=reject_if_active)- )- return self._public(stored or task)+ creation: Optional[TaskCreationResult] = None+ refusal: Optional[str] = None++ async def persist() -> None:+ nonlocal creation+ # Retain the committed result even when the caller cancels during I/O.+ creation = await self._store.create(task, reject_if_active=reject_if_active)++ try:+ await run_to_completion(persist)+ assert creation is not None+ created = self._public(creation.task)+ if not creation.created:+ return created+ if enforce_identity_admission and await self._owner_deleting(account_id, user_id):+ refusal = "Identity deletion is in progress"+ raise FailedPreconditionError(refusal)+ if admitted is not None and not await admitted():+ refusal = "Task creation is no longer admitted"+ raise FailedPreconditionError(refusal)+ except BaseException:+ if creation is not None and creation.created:+ try:+ await run_to_completion(+ lambda: self._withdraw(+ task.task_id,+ account_id=account_id,+ user_id=user_id,+ refusal=refusal,+ )+ )+ except BaseException:+ logger.exception("[TaskTracker] Failed to withdraw Task %s", task.task_id)+ raise+ return createdasync def create_if_no_running(self,@@ -743,10 +852,10 @@ class TaskTracker:account_id: Optional[str] = None,user_id: Optional[str] = None,timeout: Optional[float] = None,- poll_interval: float = _TASK_WAIT_POLL_INTERVAL,) -> TaskRecord:with _consume_wait_budget(task_id, timeout) as remaining:deadline = None if remaining is None else time.monotonic() + remaining+ next_poll_interval = _WAIT_INITIAL_POLL_INTERVALwhile True:if account_id is not None and user_id is not None:status = await self.get_task_status(@@ -775,8 +884,11 @@ class TaskTracker:if remaining is not None and remaining <= 0:raise TimeoutError(f"Timed out waiting for Task {task_id}")await asyncio.sleep(- poll_interval if remaining is None else min(poll_interval, remaining)+ next_poll_interval+ if remaining is None+ else min(next_poll_interval, remaining))+ next_poll_interval = min(_WAIT_MAX_POLL_INTERVAL, next_poll_interval * 2)async def list_tasks(self,@@ -1108,13 +1220,13 @@ class TaskTracker:user_id: Optional[str] = None,timeout: Optional[float] = None,stop_waiting: Optional[Callable[[], bool]] = None,- poll_interval: float = _TASK_WAIT_POLL_INTERVAL,) -> None:wanted = set(work_ids)if not wanted:returnwith _consume_wait_budget(task_id, timeout) as remaining:deadline = None if remaining is None else time.monotonic() + remaining+ next_poll_interval = _WAIT_INITIAL_POLL_INTERVALwhile True:works = await self._store.get_works(task_id,@@ -1133,8 +1245,11 @@ class TaskTracker:if remaining is not None and remaining <= 0:raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")await asyncio.sleep(- poll_interval if remaining is None else min(poll_interval, remaining)+ next_poll_interval+ if remaining is None+ else min(next_poll_interval, remaining))+ next_poll_interval = min(_WAIT_MAX_POLL_INTERVAL, next_poll_interval * 2)async def wait_for_task_work(self,@@ -1144,10 +1259,10 @@ class TaskTracker:account_id: Optional[str] = None,user_id: Optional[str] = None,timeout: Optional[float] = None,- poll_interval: float = _TASK_WAIT_POLL_INTERVAL,) -> None:with _consume_wait_budget(task_id, timeout) as remaining:deadline = None if remaining is None else time.monotonic() + remaining+ next_poll_interval = _WAIT_INITIAL_POLL_INTERVALwhile await self._store.has_open_work(task_id,exclude_work_id=exclude_work_id,@@ -1158,8 +1273,11 @@ class TaskTracker:if remaining is not None and remaining <= 0:raise TimeoutError(f"Timed out waiting for Work in Task {task_id}")await asyncio.sleep(- poll_interval if remaining is None else min(poll_interval, remaining)+ next_poll_interval+ if remaining is None+ else min(next_poll_interval, remaining))+ next_poll_interval = min(_WAIT_MAX_POLL_INTERVAL, next_poll_interval * 2)async def is_enqueue_complete(self,diff --git a/openviking/storage/mysql_runtime.py b/openviking/storage/mysql_runtime.pyindex 7921199b4..4090a6246 100644--- a/openviking/storage/mysql_runtime.py+++ b/openviking/storage/mysql_runtime.py@@ -247,7 +247,7 @@ def mysql_runtime_store(cls: type[T]) -> type[T]:runtime: MySQLRuntime = self._mysql_runtimeasync def invoke():- if _name in ("load", "close"):+ if _name in ("load", "initialize", "close"):loop = asyncio.get_running_loop()lifecycle = getattr(self, "_mysql_lifecycle_lock", None)if lifecycle is None or lifecycle[0] is not loop:diff --git a/tests/server/test_api_key_manager.py b/tests/server/test_api_key_manager.pyindex 90b7606eb..b7c5b1e4d 100644--- a/tests/server/test_api_key_manager.py+++ b/tests/server/test_api_key_manager.py@@ -5,6 +5,7 @@import asyncioimport hashlib+import threadingimport uuidfrom types import SimpleNamespace@@ -103,6 +104,61 @@ async def test_create_duplicate_account_raises(manager: APIKeyManager):await manager.create_account(acct, "bob")+async def test_account_reads_and_mutations_cross_loops_after_waiter_cancellation(+ manager: APIKeyManager, manager_service, monkeypatch+):+ acct = _uid()+ await manager.create_account(acct, "admin")+ agfs = manager_service.viking_fs.agfs+ entered, release, readers_waiting = threading.Event(), threading.Event(), threading.Event()+ write = agfs.write++ def delayed_write(path, *args, **kwargs):+ if path == f"/local/{acct}/_system/users.json" and not entered.is_set():+ entered.set()+ if not release.wait(timeout=10):+ raise TimeoutError("account write was not released")+ return write(path, *args, **kwargs)++ async def worker_operations():+ tasks = [+ asyncio.create_task(manager.get_account(acct)),+ asyncio.create_task(manager.get_user_membership(acct, "alice")),+ asyncio.create_task(manager.register_user(acct, "bob")),+ ]+ await asyncio.sleep(0)+ readers_waiting.set()+ return await asyncio.wait_for(asyncio.gather(*tasks), timeout=5)++ monkeypatch.setattr(agfs, "write", delayed_write)+ writer = asyncio.create_task(manager.register_user(acct, "alice"))+ worker = None+ try:+ assert await asyncio.to_thread(entered.wait, 5)+ worker = asyncio.create_task(asyncio.to_thread(lambda: asyncio.run(worker_operations())))+ assert await asyncio.to_thread(readers_waiting.wait, 5)+ cancelled = asyncio.create_task(manager.get_account(acct))+ await asyncio.sleep(0)+ cancelled.cancel()+ with pytest.raises(asyncio.CancelledError):+ await cancelled+ finally:+ release.set()+ results = await asyncio.gather(+ writer, *([worker] if worker is not None else []), return_exceptions=True+ )+ assert not any(isinstance(result, BaseException) for result in results)+ account, membership, bob_key = results[1]+ assert account["status"] == "active"+ assert membership == {"user_id": "alice", "role": Role.USER}+ assert (await manager.resolve_identity(bob_key)).user_id == "bob"+ assert {item["user_id"] for item in await _list_users(manager, acct)} == {+ "admin",+ "alice",+ "bob",+ }++async def test_ensure_trusted_identities_creates_missing_users_once(manager: APIKeyManager,):diff --git a/tests/server/test_server_health.py b/tests/server/test_server_health.pyindex 6608c76ad..9f0bf9329 100644--- a/tests/server/test_server_health.py+++ b/tests/server/test_server_health.py@@ -104,7 +104,17 @@ async def test_health_endpoint_without_api_key():@pytest.mark.parametrize(- "credentials", ["anonymous", "partial", "trusted", "keyed", "malformed", "asserted-root", "root-only", "invalid-root-only"]+ "credentials",+ [+ "anonymous",+ "partial",+ "trusted",+ "keyed",+ "malformed",+ "asserted-root",+ "root-only",+ "invalid-root-only",+ ],)async def test_health_trusted_identity_contract(credentials, caplog, service):app = create_app(@@ -288,10 +298,6 @@ async def test_ready_returns_200_after_initialized(monkeypatch):"openviking_cli.utils.ollama.detect_ollama_in_config",lambda config: (False, None, None),)- monkeypatch.setattr(- "openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance",- lambda: SimpleNamespace(embedding=SimpleNamespace(get_embedder=lambda: None)),- )app = create_app(config=ServerConfig(), service=service)transport = httpx.ASGITransport(app=app)diff --git a/tests/service/test_task_processing_time.py b/tests/service/test_task_processing_time.pyindex b3991c60d..698641c6a 100644--- a/tests/service/test_task_processing_time.py+++ b/tests/service/test_task_processing_time.py@@ -128,9 +128,7 @@ async def test_downstream_work_wait_remains_worker_occupied_time(monkeypatch):await tracker.register_work("task", "embedding", "Embedding", **OWNER)now[0] = 3- waiter = asyncio.create_task(- tracker.wait_for_work("task", ["embedding"], poll_interval=0.001, **OWNER)- )+ waiter = asyncio.create_task(tracker.wait_for_work("task", ["embedding"], **OWNER))await asyncio.sleep(0.01)now[0] = 20assert (await tracker.get("task", **OWNER)).processing_seconds == 20diff --git a/tests/service/test_task_work_contract.py b/tests/service/test_task_work_contract.pyindex 1c302138d..0c4b74b9b 100644--- a/tests/service/test_task_work_contract.py+++ b/tests/service/test_task_work_contract.py@@ -6,17 +6,220 @@ import asyncioimport jsonimport osfrom pathlib import Path+from typing import Anyfrom unittest.mock import Mock, create_autospecfrom uuid import uuid4import pytest-from openviking.service.task_domain import TaskRetentionPolicy, TaskStatus, WorkState+from openviking.service.task_domain import TaskPatch, TaskRetentionPolicy, TaskStatus, WorkStatefrom openviking.service.task_store import PersistentTaskStore, WorkRegistrationfrom openviking.service.task_tracker import TaskTracker+from openviking_cli.exceptions import FailedPreconditionErrorfrom tests.test_task_tracker import _FakeAgfs-OWNER = {"account_id": "account", "user_id": "user"}+OWNER: dict[str, Any] = {"account_id": "account", "user_id": "user"}++[email protected](params=["file", "mysql"])+async def task_stores(request):+ if request.param == "file":+ store = PersistentTaskStore(_FakeAgfs())+ yield store, store+ return+ raw = os.environ.get("OV_TEST_MYSQL")+ if not raw:+ pytest.skip("Set OV_TEST_MYSQL to exercise the shared Task/Work contract")+ sqlalchemy = pytest.importorskip("sqlalchemy.ext.asyncio")+ pytest.importorskip("aiomysql")+ from openviking.service import task_store_mysql++ params = {**json.loads(raw), "table": f"ov_contract_{uuid4().hex[:12]}"}+ first = task_store_mysql.MySQLTaskWorkStore(params=params)+ second = task_store_mysql.MySQLTaskWorkStore(params=params)+ admin = sqlalchemy.create_async_engine(first._url, **first._engine_options)+ sql_dir = Path(task_store_mysql.__file__).parent / "sql"+ schema_files = [+ sql_dir / "task_store_init.sql",+ sql_dir / "changes/0001_task_store_service_name.sql",+ ]+ try:+ async with admin.begin() as connection:+ for schema in schema_files:+ for statement in schema.read_text().replace("ov_tasks", params["table"]).split(";"):+ if statement.strip():+ await connection.exec_driver_sql(statement)+ await first.load()+ await second.load()+ yield first, second+ finally:+ await first.close()+ await second.close()+ async with admin.begin() as connection:+ await connection.run_sync(first._models.Base.metadata.drop_all)+ await admin.dispose()+++async def test_concurrent_events_preserve_history_and_lifecycle(task_stores):+ first, second = task_stores+ writer, observer = TaskTracker(first), TaskTracker(second)+ await writer.create("content_write", task_id="task", **OWNER)+ await writer.start("task", stage="parsing", **OWNER)+ initial = await observer.get_with_events("task", **OWNER)+ assert initial is not None and initial.execution_events is not None+ before = initial.execution_events+ await asyncio.gather(+ *[+ tracker.record_event(+ "task", "waiting_for_descendants", operation=f"work-{index}", **OWNER+ )+ for index, tracker in enumerate([writer, observer] * 6)+ ]+ )+ await writer.start("task", stage="parsing", **OWNER)+ await observer.patch_task("task", TaskPatch(stage="embedding"), **OWNER)+ await writer.fail("task", "provider rejected sk-testsecret123", **OWNER)+ task = await observer.get_with_events("task", **OWNER)+ assert task is not None and task.execution_events is not None+ history = task.execution_events+ items = history["items"]+ assert items[: len(before["items"])] == before["items"]+ assert [item["seq"] for item in items] == list(range(1, 19))+ assert {item["operation"] for item in items[3:15]} == {f"work-{index}" for index in range(12)}+ assert [item["kind"] for item in items[15:]] == [+ "stage_changed",+ "error_recorded",+ "status_changed",+ ]+ assert items[-1]["stage"] == "embedding"+ assert items[-1]["status"] == "failed"+ assert "[REDACTED]" in items[-2]["error"]+ assert "testsecret123" not in json.dumps(history)+ assert history["dropped_count"] == 0+ assert history["started_mid_task"] is False+ assert task.updated_at > task.created_at++[email protected]("stage_prefix", ["stage", "阶段" * 64])+async def test_shared_event_history_retains_latest_metadata_after_truncation(+ task_stores, stage_prefix+):+ from openviking.service.task_events import MAX_TASK_EVENT_BYTES, MAX_TASK_EVENTS++ first, second = task_stores+ writer, observer = TaskTracker(first), TaskTracker(second)+ await writer.create("add_resource", task_id="task", **OWNER)+ for index in range(80):+ await writer.patch_task("task", TaskPatch(stage=f"{index}:{stage_prefix}"), **OWNER)+ await writer.cancel("task", **OWNER)+ task = await observer.get_with_events("task", **OWNER)+ assert task is not None and task.execution_events is not None+ history = task.execution_events+ items = history["items"]+ assert len(items) <= MAX_TASK_EVENTS+ assert len(json.dumps(history).encode()) <= MAX_TASK_EVENT_BYTES+ assert history["dropped_count"] + len(items) == 83+ assert [item["seq"] for item in items] == list(range(history["dropped_count"] + 1, 84))+ assert [item["status"] for item in items[-2:]] == ["cancelling", "cancelled"]+ assert history["started_mid_task"] is False+++async def test_store_withdraws_only_tasks_without_registered_work(task_stores):+ first, second = task_stores+ tracker = TaskTracker(first)+ await tracker.create("compile", task_id="unused", **OWNER)+ assert (await second.withdraw_unaccepted("unused", **OWNER)).is_applied+ assert await tracker.get("unused", **OWNER) is None++ await tracker.create("compile", task_id="accepted", **OWNER)+ assert (+ await tracker.register_work("accepted", "work", "ExternalTask", **OWNER)+ is WorkRegistration.CREATED+ )+ assert not (await second.withdraw_unaccepted("accepted", **OWNER)).accepted+ assert await tracker.get("accepted", **OWNER) is not None+ assert await tracker.get_work_state("accepted", "work", **OWNER) is WorkState.PENDING+++async def test_withdrawal_is_atomic_with_work_registration(task_stores):+ first, second = task_stores+ tracker = TaskTracker(first)+ await tracker.create("compile", task_id="task", **OWNER)++ withdrawn, registered = await asyncio.gather(+ second.withdraw_unaccepted("task", **OWNER),+ tracker.register_work("task", "work", "ExternalTask", **OWNER),+ )+ if withdrawn.is_applied:+ assert registered is WorkRegistration.REJECTED+ assert await tracker.get("task", **OWNER) is None+ else:+ assert not withdrawn.accepted+ assert registered is WorkRegistration.CREATED+ assert await tracker.get_work_state("task", "work", **OWNER) is WorkState.PENDING++[email protected]("postcheck", ["query-error", "refused"])+async def test_creator_postcheck_preserves_or_cancels_accepted_task(monkeypatch, postcheck):+ store = PersistentTaskStore(_FakeAgfs())+ creator, replay = TaskTracker(store), TaskTracker(store)+ inserted, release = asyncio.Event(), asyncio.Event()+ create = store.create+ checks = 0+ query_error = ConnectionError("Identity storage unavailable")++ async def guard(account_id, user_id):+ nonlocal checks+ checks += 1+ if checks == 2:+ if postcheck == "query-error":+ raise query_error+ return True+ return False++ async def delayed_create(task, **kwargs):+ result = await create(task, **kwargs)+ inserted.set()+ await release.wait()+ return result++ creator.set_deletion_guard(guard)+ monkeypatch.setattr(store, "create", delayed_create)+ request = asyncio.create_task(creator.create("compile", task_id="task", **OWNER))+ try:+ await asyncio.wait_for(inserted.wait(), 1)+ await replay.create("compile", task_id="task", **OWNER)+ assert (+ await replay.register_work("task", "work", "ExternalTask", **OWNER)+ is WorkRegistration.CREATED+ )+ release.set()+ with pytest.raises(+ ConnectionError if postcheck == "query-error" else FailedPreconditionError+ ) as raised:+ await asyncio.wait_for(request, 1)+ task = await replay.get("task", **OWNER)+ assert task is not None+ assert await replay.get_work_state("task", "work", **OWNER) is WorkState.PENDING++ if postcheck == "query-error":+ assert raised.value is query_error+ assert task.status is TaskStatus.PENDING+ await replay.cancel("task", **OWNER)+ else:+ assert task.status is TaskStatus.CANCELLING++ await replay.mark_work_done("task", "work", **OWNER)+ assert (await replay.wait("task", timeout=0, **OWNER)).status is TaskStatus.CANCELLED+ finally:+ release.set()+ request.cancel()+ await asyncio.gather(request, return_exceptions=True)@pytest.fixture@@ -43,7 +246,7 @@ async def test_task_work_wait_includes_recovered_and_later_enqueued_work(trackerawait tracker.register_work("task", "root", "Inline", **OWNER)await tracker.reconcile_delivery("task", "semantic", "Semantic", **OWNER)waiting = asyncio.create_task(- tracker.wait_for_task_work("task", exclude_work_id="root", poll_interval=0.001, **OWNER)+ tracker.wait_for_task_work("task", exclude_work_id="root", **OWNER))try:await asyncio.sleep(0)@@ -199,6 +402,15 @@ async def test_mysql_requeue_and_concurrent_completion_are_durable(deletion):assert isolated_task is not Noneawait isolated.register_work(task_id, "semantic", "Semantic", **OWNER)+ assert (await second.withdraw_unaccepted(task_id, **OWNER)).is_applied+ assert await observer.get(task_id, **OWNER) is None+ assert await isolated.get(task_id, **OWNER) is not None+ assert await isolated.get_work_state(task_id, "semantic", **OWNER) is WorkState.PENDING+ assert (+ await writer.create_if_no_running("content_write", uri, task_id=task_id, **OWNER)+ is not None+ )+await writer.start(task_id, **OWNER)await writer.register_work(task_id, "semantic", "Semantic", **OWNER)await observer.register_work(task_id, "embedding", "Embedding", **OWNER)diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.pyindex 34d1df5e5..1d724a4ce 100644--- a/tests/test_task_tracker.py+++ b/tests/test_task_tracker.py@@ -17,6 +17,7 @@ from openviking.server.identity import RequestContext, Rolefrom openviking.service.session_service import SessionServicefrom openviking.service.task_domain import TaskPatchfrom openviking.service.task_store import (+ SYSTEM_TASK_ACCOUNT_ID,PersistentTaskStore,_task_from_payload,_task_to_payload,@@ -28,9 +29,10 @@ from openviking.service.task_tracker import (get_task_tracker,run_tracked_operation,set_task_tracker,+ task_admission,)from openviking.service.task_work_index import get_task_context-from openviking_cli.exceptions import DeadlineExceededError+from openviking_cli.exceptions import DeadlineExceededError, FailedPreconditionErrorfrom openviking_cli.session.user_id import UserIdentifierpytestmark = pytest.mark.asyncio@@ -177,6 +179,178 @@ async def test_create_task(tracker: TaskTracker):assert task.status == TaskStatus.PENDING+async def test_deletion_guard_refuses_new_tasks_for_deleting_identities(tracker: TaskTracker):+ existing = await tracker.create("add_resource", task_id="replayed", **_owner_kwargs())+ deleting = {("acme", "alice")}++ async def guard(account_id, user_id):+ return (account_id, user_id) in deleting++ tracker.set_deletion_guard(guard)++ with pytest.raises(FailedPreconditionError):+ await tracker.create("add_resource", task_id="late", **_owner_kwargs())+ assert await tracker.get("late", **_owner_kwargs()) is None+ replayed = await tracker.create("add_resource", task_id="replayed", **_owner_kwargs())+ assert replayed.task_id == existing.task_id+ assert await tracker.create("add_resource", **_owner_kwargs(user_id="bob"))+ assert await tracker.create("account_delete", account_id=SYSTEM_TASK_ACCOUNT_ID, user_id="root")+++async def test_identity_cleanup_task_creation_allows_deleting_owner(tracker: TaskTracker):+ async def guard(account_id, user_id):+ return (account_id, user_id) == ("acc", "alice")++ tracker.set_deletion_guard(guard)++ task = await tracker.create(+ "user_delete",+ task_id="cleanup",+ account_id="acc",+ user_id="alice",+ )+ assert task.task_id == "cleanup"+ with pytest.raises(FailedPreconditionError):+ await tracker.create("add_resource", account_id="acc", user_id="alice")+++async def test_wait_uses_short_initial_poll(monkeypatch, tracker: TaskTracker):+ task = await tracker.create("add_resource", task_id="task", **_owner_kwargs())+ delays = []++ async def sleep(delay):+ delays.append(delay)+ await tracker.complete(task.task_id, {}, **_owner_kwargs())++ monkeypatch.setattr(asyncio, "sleep", sleep)++ assert (await tracker.wait(task.task_id, **_owner_kwargs())).status is TaskStatus.COMPLETED+ assert delays == [0.05]+++async def test_wait_for_task_work_uses_short_initial_poll(monkeypatch, tracker: TaskTracker):+ task = await tracker.create("add_resource", task_id="task", **_owner_kwargs())+ await tracker.start(task.task_id, **_owner_kwargs())+ await tracker.register_work(task.task_id, "work", "Semantic", **_owner_kwargs())+ delays = []++ async def sleep(delay):+ delays.append(delay)+ await tracker.mark_work_done(task.task_id, "work", **_owner_kwargs())++ monkeypatch.setattr(asyncio, "sleep", sleep)++ await tracker.wait_for_task_work(task.task_id, **_owner_kwargs())+ assert delays == [0.05]++[email protected]("failure", ["deletion", "identity-query", "watch-admission"])+async def test_creation_rejected_after_insert_withdraws_the_task(tracker, failure):+ deleting = set()+ store_create = tracker._store.create+ error = ConnectionError("Admission storage unavailable")++ async def guard(account_id, user_id):+ if (account_id, user_id) not in deleting:+ return False+ if failure == "identity-query":+ raise error+ return failure == "deletion"++ async def admitted():+ if deleting and failure == "watch-admission":+ raise error+ return True++ async def create_after_cleanup(task, **kwargs):+ # The check has passed; deletion fences the identity and finishes+ # listing Tasks before this insert lands.+ deleting.add((task.account_id, task.user_id))+ assert await tracker.list_tasks(**_owner_kwargs(), limit=None) == []+ return await store_create(task, **kwargs)++ tracker.set_deletion_guard(guard)+ tracker._store.create = create_after_cleanup++ with task_admission("late", admitted):+ with pytest.raises(+ FailedPreconditionError if failure == "deletion" else ConnectionError+ ) as raised:+ await tracker.create(+ "connector_import", resource_id="resource", task_id="late", **_owner_kwargs()+ )+ if failure != "deletion":+ assert raised.value is error+ assert await tracker.get("late", **_owner_kwargs()) is None+ tracker._store.create = store_create+ deleting.clear()+ assert await tracker.create_if_no_running("connector_import", "resource", **_owner_kwargs())++[email protected]("cancel_at", ["insert", "identity-query", "watch-admission"])+async def test_cancel_during_task_creation_withdraws_the_unaccepted_task(cancel_at):+ entered = asyncio.Event()+ release = threading.Event()+ loop = asyncio.get_running_loop()++ class BlockingAgfs(_FakeAgfs):+ def write(self, path, data, *args, **kwargs):+ if cancel_at == "insert" and json.loads(data)["status"] == "pending":+ loop.call_soon_threadsafe(entered.set)+ assert release.wait(3)+ return super().write(path, data, *args, **kwargs)++ agfs = BlockingAgfs()+ tracker = TaskTracker(PersistentTaskStore(agfs))+ checks = 0++ async def check():+ nonlocal checks+ checks += 1+ if checks == 2:+ entered.set()+ await asyncio.Event().wait()++ async def guard(account_id, user_id):+ if cancel_at == "identity-query":+ await check()+ return False++ async def admitted():+ if cancel_at == "watch-admission":+ await check()+ return True++ tracker.set_deletion_guard(guard)+ with task_admission("late", admitted):+ running = asyncio.create_task(+ tracker.create("connector_import", task_id="late", **_owner_kwargs())+ )+ try:+ await asyncio.wait_for(entered.wait(), 1)+ running.cancel()+ await asyncio.sleep(0)+ running.cancel()+ finally:+ release.set()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(running, 1)++ assert await TaskTracker(PersistentTaskStore(agfs)).list_tasks(**_owner_kwargs()) == []+++async def test_admission_gates_only_its_own_task_id(tracker: TaskTracker):+ async def refused():+ return False++ with task_admission("run-1", refused):+ with pytest.raises(FailedPreconditionError):+ await tracker.create("add_resource", task_id="run-1", **_owner_kwargs())+ assert await tracker.create("add_resource", task_id="other", **_owner_kwargs())+ assert await tracker.get("run-1", **_owner_kwargs()) is None+ assert await tracker.create("add_resource", task_id="run-1", **_owner_kwargs())++async def test_start_task(tracker: TaskTracker):task = await tracker.create("session_commit", **_owner_kwargs())await tracker.start(task.task_id)diff --git a/tests/test_task_tracker_concurrency.py b/tests/test_task_tracker_concurrency.pyindex 941f91b8e..f4a5253e3 100644--- a/tests/test_task_tracker_concurrency.py+++ b/tests/test_task_tracker_concurrency.py@@ -265,29 +265,56 @@ async def test_keyed_lock_registry_is_cleaned_after_waiter_cancellation():@pytest.mark.asyncio-async def test_file_store_concurrent_fixed_id_create_is_idempotent():[email protected]("failure", ["none", "identity-query", "cancel"])+async def test_file_store_concurrent_fixed_id_create_is_idempotent(failure):from openviking.service.task_store import PersistentTaskStore- from openviking.service.task_tracker import TaskTracker+ from openviking.service.task_tracker import TaskStatus, TaskTrackerfrom tests.test_task_tracker import _FakeAgfs- tracker = TaskTracker(PersistentTaskStore(_FakeAgfs()))- first, second = await asyncio.gather(- tracker.create(- "add_resource",- task_id="fixed",- account_id="acme",- user_id="alice",- ),- tracker.create(- "add_resource",- task_id="fixed",- account_id="acme",- user_id="alice",- ),+ store = PersistentTaskStore(_FakeAgfs())+ writer, replay = TaskTracker(store), TaskTracker(store)+ entered, release = asyncio.Event(), asyncio.Event()+ create = store.create++ async def delayed_create(task, **kwargs):+ if task.meta.get("delayed"):+ entered.set()+ await release.wait()+ return await create(task, **kwargs)++ store.create = delayed_create+ checks = 0++ async def guard(account_id, user_id):+ nonlocal checks+ checks += 1+ if checks > 1 and failure == "identity-query":+ raise ConnectionError("Identity storage unavailable")+ return False++ replay.set_deletion_guard(guard)+ owner = {"account_id": "acme", "user_id": "alice"}+ duplicate = asyncio.create_task(+ replay.create("add_resource", task_id="fixed", meta={"delayed": True}, **owner))-- assert first.task_id == second.task_id == "fixed"- assert len(await tracker.list_tasks(account_id="acme", user_id="alice")) == 1+ try:+ await asyncio.wait_for(entered.wait(), 1)+ first = await writer.create("add_resource", task_id="fixed", **owner)+ await writer.start(first.task_id, **owner)+ await writer.register_work(first.task_id, "work", "Semantic", **owner)+ if failure == "cancel":+ duplicate.cancel()+ finally:+ release.set()+ if failure == "cancel":+ with pytest.raises(asyncio.CancelledError):+ await duplicate+ else:+ second = await duplicate+ assert first.task_id == second.task_id == "fixed"+ assert (await writer.get("fixed", **owner)).status is TaskStatus.RUNNING+ assert await writer.has_work("fixed", **owner)+ assert len(await writer.list_tasks(**owner)) == 1@pytest.mark.asyncio