Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
feat(watch): Watch target kind job waits on Avibe Agent job handles (#2343)
Call White013cf5e82e12
diff --git a/core/watches.py b/core/watches.pyindex 13c7c65d0..b8f82e105 100644--- a/core/watches.py+++ b/core/watches.py@@ -12,7 +12,7 @@ from dataclasses import asdict, dataclass, field, replacefrom datetime import datetime, timezonefrom functools import partialfrom pathlib import Path-from typing import Any, Optional+from typing import Any, Mapping, Optionalfrom uuid import uuid4from config import paths@@ -41,6 +41,7 @@ from core.runtime_work import ()from core.scheduled_tasks import TaskExecutionRequest, TaskExecutionStorefrom storage.background import (+ COMMAND_TIMED_OUT_METADATA_KEY,DEFINITION_CYCLE_COLUMNS,NO_EVENT_EXIT_CODE,WATCH_CIRCUIT_BREAKER_METADATA_KEY as CIRCUIT_BREAKER_METADATA_KEY,@@ -51,10 +52,13 @@ from storage.background import (WATCH_HOOK_OUTCOME_WAITER_FAILURE,WATCH_LIFETIME_STARTED_AT_METADATA_KEY as LIFETIME_STARTED_AT_METADATA_KEY,WATCH_RECENT_EVENT_TIMESTAMPS_METADATA_KEY as RECENT_EVENT_TIMESTAMPS_METADATA_KEY,+ WATCH_TARGET_KIND_JOB,+ WATCH_TARGET_METADATA_KEY,DefinitionWriteConflict,DefinitionWriteExpectation,SQLiteBackgroundTaskStore,definition_resume_clear_columns,+ watch_job_target,watch_metadata_after_resume,)from vibe import runtime@@ -129,6 +133,29 @@ WATCH_STORE_RECONCILE_PAUSE_FAILURES = 3WATCH_STORE_RETRY_INITIAL_SECONDS = 30.0WATCH_STORE_RETRY_MAX_SECONDS = 600.0WATCH_RECOVERY_ENTRY_TIMEOUT_SECONDS = 2 * DEFAULT_PROCESS_TERMINATE_TIMEOUT_SECONDS+#: How often a job Watch looks at its job handle. The job host enforces the job's+#: own deadline on every look (J3), so this also bounds how late a deadline can be+#: enforced while no wrapper does.+WATCH_JOB_POLL_SECONDS = 1.0+#: Why a Watch stopped the job it owns (the job host records the first reason given).+JOB_STOP_WATCH_REMOVED = "watch_removed"+JOB_STOP_WATCH_DISABLED = "watch_disabled"+JOB_STOP_WATCH_LIFETIME = "watch_lifetime"+JOB_STOP_SESSION_ARCHIVED = "session_archived"+JOB_STOP_VIBE_STOP = "vibe_stop"+JOB_STOP_UNREADABLE = "watch_unreadable_state"+++class JobWatchUpdateRefused(ValueError):+ """A job Watch takes only a new name, message, or lifetime: its job fixed the rest."""++ code = "job_watch_update_refused"++ def __init__(self, watch_id: str) -> None:+ super().__init__(+ f"watch {watch_id} waits on a running command: only its name, message, and lifetime can be changed"+ )+ self.watch_id = watch_iddef _publish_watch_definitions_updated() -> None:@@ -252,6 +279,12 @@ class ManagedWatch:last_exit_code: Optional[int] = Nonemetadata: dict[str, Any] = field(default_factory=dict)+ @property+ def job_target(self) -> Optional[dict[str, Any]]:+ """The job handle this Watch waits on, or ``None`` for a waiter-command Watch."""++ return watch_job_target(self.metadata)+@propertydef last_cycle_outcome(self) -> tuple[Optional[int], Optional[str]]:"""Return the last completed cycle's exit code and error as one snapshot."""@@ -266,6 +299,9 @@ class ManagedWatch:@classmethoddef from_dict(cls, payload: dict[str, Any]) -> "ManagedWatch":+ metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {}+ # A job Watch has no waiter, so no waiter exit code is ever retried.+ default_retry_codes = [] if watch_job_target(metadata) else [DEFAULT_RETRY_EXIT_CODE]return cls(id=str(payload.get("id") or uuid4().hex[:12]),name=(str(payload["name"]).strip() if payload.get("name") is not None else None) or None,@@ -281,7 +317,7 @@ class ManagedWatch:mode=str(payload.get("mode") or "once"),timeout_seconds=_payload_float(payload, "timeout_seconds", 21600.0),lifetime_timeout_seconds=_payload_float(payload, "lifetime_timeout_seconds", 0.0),- retry_exit_codes=[int(code) for code in (payload.get("retry_exit_codes") or [DEFAULT_RETRY_EXIT_CODE])],+ retry_exit_codes=[int(code) for code in (payload.get("retry_exit_codes") or default_retry_codes)],retry_delay_seconds=_payload_float(payload, "retry_delay_seconds", 30.0),post_to=payload.get("post_to"),deliver_key=payload.get("deliver_key"),@@ -294,7 +330,7 @@ class ManagedWatch:last_event_at=payload.get("last_event_at"),last_error=payload.get("last_error"),last_exit_code=(int(payload["last_exit_code"]) if payload.get("last_exit_code") is not None else None),- metadata=payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {},+ metadata=metadata,)@@ -718,6 +754,178 @@ class ManagedWatchStore:expected_reference_agent_id=expected_reference_agent_id,)+ def adopt_or_create_job_watch(+ self,+ job: Mapping[str, Any],+ *,+ agent_name: Optional[str] = None,+ session_key: str = "",+ post_to: Optional[str] = None,+ deliver_key: Optional[str] = None,+ message: Optional[str] = None,+ metadata: Optional[dict[str, Any]] = None,+ user_context: Any = None,+ ) -> str:+ """The id of the Watch that owns ``job`` (its ``meta.json``), created if none does.++ Recovery invariant J6: keyed by ``job_id``, so a hand-over repeated after a+ crash gets the same Watch back, even one removed since. The Watch is a+ ``once`` Watch on the job's Session, delivered through Watch's existing+ follow-up path; its cycle waits on the job and never runs anything.++ Raising means no Watch owns the job, so it raises only once a lookup has+ confirmed that: the owner is looked up before anything that could refuse a new+ Watch (the caller's access, the Agent, the job record's shape), and again+ before a refusal escapes, in case a concurrent hand-over created it meanwhile.+ Once the row is committed this returns its id: a failed mirror refresh+ afterwards is logged, never raised.+ """++ job_id = job.get("job_id")+ owner = self.find_job_watch(job_id)+ if owner is not None:+ return owner+ try:+ return self._create_job_watch(+ job,+ agent_name=agent_name,+ session_key=session_key,+ post_to=post_to,+ deliver_key=deliver_key,+ message=message,+ metadata=metadata,+ user_context=user_context,+ )+ except Exception:+ try:+ owner = self.find_job_watch(job_id)+ except Exception:+ owner = None+ if owner is not None:+ return owner+ raise++ def _create_job_watch(+ self,+ job: Mapping[str, Any],+ *,+ agent_name: Optional[str],+ session_key: str,+ post_to: Optional[str],+ deliver_key: Optional[str],+ message: Optional[str],+ metadata: Optional[dict[str, Any]],+ user_context: Any,+ ) -> str:+ from core.vibe_agents import ensure_agent_name_access+ from storage.resource_access_service import (+ ensure_harness_definition_write,+ metadata_with_resource_user_context,+ )++ ensure_harness_definition_write(user_context)+ ensure_agent_name_access(agent_name, user_context=user_context)+ target = _job_target_from_meta(job)+ watch_metadata = metadata_with_resource_user_context(metadata, user_context)+ watch_metadata[WATCH_TARGET_METADATA_KEY] = target+ watch = ManagedWatch(+ id=uuid4().hex[:12],+ name=None,+ session_key=session_key,+ session_id=str(job["session_id"]),+ agent_name=agent_name,+ session_policy="existing",+ command=[],+ # What the read model shows for a job Watch; never written as a waiter.+ shell_command=target["command"] or None,+ message=message,+ cwd=str(job.get("cwd") or "") or None,+ mode="once",+ timeout_seconds=0.0,+ retry_exit_codes=[],+ retry_delay_seconds=0.0,+ post_to=post_to,+ deliver_key=deliver_key,+ metadata=watch_metadata,+ )+ watch.metadata[LIFETIME_STARTED_AT_METADATA_KEY] = watch.created_at+ if self._sqlite is not None:+ watch_id = self._sqlite.adopt_or_create_job_watch(watch.to_dict())+ try:+ self.load()+ except Exception:+ logger.warning(+ "Job %s is Watch %s, but the watch mirror could not be refreshed",+ target["job_id"],+ watch_id,+ exc_info=True,+ )+ else:+ with self._mirror_lock:+ owner = self.find_job_watch(target["job_id"])+ if owner is not None:+ return owner+ self._watches[watch.id] = watch+ try:+ self._save()+ except Exception:+ self._reload_after_lost_write(watch.id)+ raise+ watch_id = watch.id+ _publish_watch_definitions_updated()+ return watch_id++ def find_job_watch(self, job_id: Any) -> Optional[str]:+ """The id of the Watch that owns or owned ``job_id`` (J6), or ``None``."""++ if self._sqlite is not None:+ return self._sqlite.find_job_watch(job_id)+ with self._mirror_lock:+ for watch in self._watches.values():+ target = watch.job_target+ if target is not None and target["job_id"] == job_id:+ return watch.id+ return None++ def list_job_watch_targets(self) -> list[dict[str, Any]]:+ """Job Watches whose job may still need its owner (``SQLiteBackgroundTaskStore.list_job_watch_targets``).++ The file backend has no removed rows to report: a removed file-backed Watch is gone.+ """++ if self._sqlite is not None:+ return self._sqlite.list_job_watch_targets()+ with self._mirror_lock:+ return [+ {"id": watch.id, "enabled": watch.enabled, "deleted": False, "session_archived": False, "target": target}+ for watch in self._watches.values()+ if (target := watch.job_target) is not None and not target.get("released_at")+ ]++ def job_watch_settled(self, job_id: str) -> bool:+ """The Watch half of J5 for ``LocalJobHost.prune``: no Watch still manages ``job_id``."""++ if self._sqlite is not None:+ return self._sqlite.job_watch_settled(job_id)+ return all(target["job_id"] != job_id for target in (row["target"] for row in self.list_job_watch_targets()))++ def release_job_watch(self, watch_id: str, *, released_at: str) -> None:+ """Record that ``watch_id`` has no job left to manage, so the ownership sweep stops visiting it."""++ with self._mirror_lock:+ if self._sqlite is not None:+ self._sqlite.release_job_watch(watch_id, released_at=released_at)+ cached = self._watches.get(watch_id)+ target = cached.job_target if cached is not None else None+ if cached is None or target is None:+ return+ cached.metadata = {+ **cached.metadata,+ WATCH_TARGET_METADATA_KEY: {**target, "released_at": released_at},+ }+ if self._sqlite is None:+ self._save()+def remove_watch(self, watch_id: str) -> bool:"""Delete a watch; the mirror rolls back with the delete (HFR-275).@@ -747,16 +955,18 @@ class ManagedWatchStore:watch = self._watches[watch_id]expect = self._read_state(watch)if enabled and not watch.enabled:+ # First, because it may refuse (a job Watch): nothing is mutated yet.+ resumed_metadata = watch_metadata_after_resume(+ watch.metadata,+ resumed_at=_utc_now_iso(),+ )# Same field split the storage layer applies to the Harness UI's# toggle, so the two doorways cannot drift apart again.self._clear_cycle_state(watch,definition_resume_clear_columns("watch", watch.mode),)- watch.metadata = watch_metadata_after_resume(- watch.metadata,- resumed_at=_utc_now_iso(),- )+ watch.metadata = resumed_metadatawatch.enabled = enabledwatch.updated_at = _utc_now_iso()if not self._write_watch(watch, expect):@@ -805,6 +1015,28 @@ class ManagedWatchStore:# Captured before the first mutation: the state ``vibe watch update`` read and# resolved its payload from.expect = self._read_state(watch)+ if watch.job_target is not None:+ # A job Watch is a read-only projection of its job: the hand-over fixed+ # its Session, Agent, delivery, and waiter. Only what it says and how+ # long it may wait can change (an allowlist, so a new field is fixed too).+ requested = {+ "session_key": session_key,+ "session_id": session_id,+ "agent_name": agent_name,+ "session_policy": session_policy if session_policy is not None else watch.session_policy,+ "post_to": post_to,+ "deliver_key": deliver_key,+ "metadata": metadata if metadata is not None else watch.metadata,+ "command": command,+ "shell_command": shell_command,+ "cwd": cwd,+ "mode": mode,+ "timeout_seconds": timeout_seconds,+ "retry_exit_codes": retry_exit_codes,+ "retry_delay_seconds": retry_delay_seconds,+ }+ if any(value != getattr(watch, field) for field, value in requested.items()):+ raise JobWatchUpdateRefused(watch_id)waiter_lifecycle_changed = (mode != watch.modeor command != watch.command@@ -1149,6 +1381,7 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):await self.service._persist_runtime_state()return Trueself.service._note_store_reconcile_ok()+ await self.service._settle_unowned_jobs()if self.service._runtime_state_dirty:await self.service._persist_runtime_state()if self.service._runtime_state_dirty:@@ -1645,6 +1878,7 @@ class ManagedWatchService:if self._runtime_state_dirty:self._write_runtime_state()self._note_store_reconcile_ok()+ await self._settle_unowned_jobs()except asyncio.CancelledError:raiseexcept Exception as exc:@@ -2139,6 +2373,8 @@ class ManagedWatchService:*,blocking_run_id: str | None = None,) -> bool:+ if watch.job_target is not None:+ return await self._stop_job_at_lifetime_async(watch, blocking_run_id=blocking_run_id)kwargs: dict[str, Any] = {}if blocking_run_id is None:kwargs = {@@ -2164,6 +2400,180 @@ class ManagedWatchService:**kwargs,)+ async def _wait_for_job(self, watch: ManagedWatch, *, lifetime_started: float) -> None:+ """A job Watch's cycle: wait on the job handle until the job ends; never run anything.++ The handle outlives this process, so after a restart the next cycle re-attaches+ to the same job and reports it once. No pid is registered for this Watch: the+ stale-worker reaping at startup must never touch a job. The job host enforces+ the job's own deadline on every look (J3); the Watch's lifetime is enforced+ here; removal, pausing, and archive stop the job in ``_settle_unowned_jobs``,+ never through this task's cancellation, which a restart causes too.+ """++ while self._running:+ if not self._owns_service_instance():+ return+ current = self.store.get_watch(watch.id)+ if current is None or not current.enabled or current.job_target is None:+ return+ watch = current+ lifetime_remaining: Optional[float] = None+ if watch.lifetime_timeout_seconds > 0:+ lifetime_remaining = watch.lifetime_timeout_seconds - (+ asyncio.get_running_loop().time() - lifetime_started+ )+ if lifetime_remaining <= 0:+ await self._commit_lifetime_timeout_async(watch)+ return+ target = watch.job_target+ try:+ status = await _job_host(target).wait(target["job_id"], deadline_s=0)+ except KeyError:+ status = None # the job record is gone: report what is known+ except ValueError as exc:+ # Unreadable state (its deadline, say): the Watch cannot keep its promises+ # for this job, so it stops the command, and lets go only once the job+ # host confirms nothing of it runs.+ if await _stop_job(target, JOB_STOP_UNREADABLE):+ await self._commit_job_result_async(watch)+ else:+ await self._hold_unmanageable_job_async(watch, exc)+ return+ except Exception as exc:+ # Possibly transient (I/O): keep the job, say so, and look again later.+ await self._hold_unmanageable_job_async(watch, exc)+ return+ if status is None or status.state != "running":+ await self._commit_job_result_async(watch)+ return+ delay = WATCH_JOB_POLL_SECONDS+ if lifetime_remaining is not None:+ delay = min(delay, lifetime_remaining)+ await asyncio.sleep(delay)++ async def _stop_job_at_lifetime_async(+ self,+ watch: ManagedWatch,+ *,+ blocking_run_id: str | None,+ ) -> bool:+ """The Watch's lifetime ended: stop the job it owns, then report how the job ended."""++ target = watch.job_target+ if not await _stop_job(target, JOB_STOP_WATCH_LIFETIME):+ # Retiring the Watch now would leave the job running with no owner.+ return await self._hold_unmanageable_job_async(+ watch, RuntimeError(f"job {target['job_id']} could not be stopped at the Watch's lifetime")+ )+ if blocking_run_id is None:+ return await self._commit_job_result_async(watch)+ # As for every Watch: while an earlier follow-up is still in flight, no second+ # report is queued; the stamp says why.+ return await self._commit_cycle_result_async(+ watch,+ exit_code=None,+ error=self._t(+ "harness.watch.lifetimeExpiredWithActiveFollowUp",+ run_id=blocking_run_id,+ ),+ disable=True,+ metadata_updates={+ WATCH_TARGET_METADATA_KEY: {**target, "released_at": _utc_now_iso()},+ },+ )++ async def _commit_job_result_async(self, watch: ManagedWatch) -> bool:+ """Report how the job ended through Watch's follow-up and retire the Watch, in one commit.++ The command's exit code goes into the report and the target, never into+ ``last_exit_code``: every reader of that column treats it as a waiter's code,+ where 64 means "no event" and 124 a timeout.+ """++ target = watch.job_target+ try:+ outcome = await asyncio.to_thread(_job_outcome, watch, target)+ except KeyError:+ outcome = _missing_job_outcome(watch, target)+ except _JobStillRunning as exc:+ # Never reported as ended: whoever called this had no proof it ended.+ return await self._hold_unmanageable_job_async(watch, exc)+ except Exception as exc:+ # The outcome cannot be read. The Watch lets go only once nothing of the job runs.+ if not await _stop_job(target, JOB_STOP_UNREADABLE):+ return await self._hold_unmanageable_job_async(watch, exc)+ outcome = _unreadable_job_outcome(watch, target)+ return await self._commit_cycle_result_async(+ watch,+ exit_code=None,+ error=self._t(outcome.error_key, **outcome.error_args) if outcome.error_key else None,+ event_detected=outcome.event_detected,+ disable=True,+ prompt=_build_prompt(watch.message or watch.prefix, outcome.report),+ metadata_updates={+ WATCH_TARGET_METADATA_KEY: {+ **target,+ "exit_code": outcome.exit_code,+ "released_at": _utc_now_iso(),+ },+ COMMAND_TIMED_OUT_METADATA_KEY: outcome.timed_out,+ },+ )++ async def _hold_unmanageable_job_async(self, watch: ManagedWatch, exc: BaseException) -> bool:+ """The Watch can neither follow nor stop its job: it keeps it, says so, and tries again later.++ Never released: letting go of a job that may still run would leave it with no+ owner. The stamp keeps the Watch enabled and queues no follow-up.+ """++ logger.warning(+ "Watch %s cannot follow or stop job %s; it keeps the job and retries: %s",+ watch.id,+ watch.job_target["job_id"],+ exc,+ )+ await self._commit_cycle_result_async(+ watch,+ exit_code=None,+ error=self._t("harness.watch.jobUnmanageable"),+ disable=False,+ )+ self._back_off_watch_after_store_error("hold an unmanageable job", exc, watch_id=watch.id)+ return False++ async def _settle_unowned_jobs(self) -> None:+ """Stop the job of every job Watch that no longer owns it, then release that Watch.++ A Watch owns its job while it is enabled. Removal, pausing (``/new`` and a+ refused remote origin included), and Session archive all leave the row not+ enabled -- written by the CLI, the Web UI, or a teardown, in any process, maybe+ while this service was down -- so ownership is reconciled here from the rows+ rather than at each of those doorways.+ """++ if not self._owns_service_instance():+ return+ try:+ rows = await self._run_runtime_sync(self.store.list_job_watch_targets)+ except Exception:+ logger.warning("Could not read job Watches to settle their jobs", exc_info=True)+ return+ for row in rows:+ if _job_still_owned(row):+ continue+ if not await _stop_job(row["target"], _unowned_job_stop_reason(row)):+ continue+ try:+ await self._run_runtime_sync(+ self.store.release_job_watch,+ row["id"],+ released_at=_utc_now_iso(),+ )+ except Exception:+ logger.warning("Could not release job Watch %s", row["id"], exc_info=True)+async def _sleep_before_retry(self,watch: ManagedWatch,@@ -2264,6 +2674,9 @@ class ManagedWatchService:guarded=True,):return+ if watch.job_target is not None:+ await self._wait_for_job(watch, lifetime_started=lifetime_started)+ returncwd_error = _missing_watch_cwd_error(watch)if cwd_error:await self._stop_watch_for_missing_cwd_async(@@ -2690,6 +3103,322 @@ class ManagedWatchService:return False+def agent_jobs_dir() -> Path:+ """Where the Avibe Agent's jobs live (plan section 5.3): the adapter's ``LocalJobHost`` hosts them here."""++ return paths.get_state_dir() / "agent_core" / "jobs"+++def _job_target_from_meta(job: Mapping[str, Any]) -> dict[str, Any]:+ """The Watch target for a job's ``meta.json`` (``job.schema.json``); ``ValueError`` if it cannot be one."""++ job_id = job.get("job_id")+ state_dir = job.get("state_dir")+ command = job.get("command")+ if not isinstance(job_id, str) or not job_id.startswith("job_"):+ raise ValueError("a job Watch needs the job's job_id")+ if not isinstance(state_dir, str) or not os.path.isabs(state_dir):+ raise ValueError("a job Watch needs the job's absolute state_dir")+ if os.path.basename(os.path.normpath(state_dir)) != job_id:+ raise ValueError("the job's state_dir does not name its job_id")+ if not isinstance(command, str):+ raise ValueError("a job Watch needs the job's command")+ if not str(job.get("session_id") or "").strip():+ raise ValueError("a job Watch needs the job's session_id")+ return {"kind": WATCH_TARGET_KIND_JOB, "job_id": job_id, "state_dir": state_dir, "command": command}+++def _job_host(target: Mapping[str, Any]):+ """The job host over the directory that holds ``target``'s job (``LocalJobHost``, PR #2337)."""++ from core.agent_core.tools.jobs import LocalJobHost++ state_dir = os.path.normpath(str(target.get("state_dir") or ""))+ if not os.path.isabs(state_dir) or os.path.basename(state_dir) != target["job_id"]:+ raise ValueError(f"job Watch target has no usable job directory: {state_dir!r}")+ return LocalJobHost(os.path.dirname(state_dir))+++async def _stop_job(target: Mapping[str, Any], reason: str) -> bool:+ """Kill ``target``'s process tree through its job host with ``reason``.++ ``True`` only with proof that nothing of the job runs: its record is gone, or the+ job host's status says it is not running. Anything else, a failed kill included,+ is ``False``, and the caller must keep the job.+ """++ job_id = target["job_id"]+ try:+ host = _job_host(target)+ except Exception:+ logger.warning("Job %s has no usable job directory; it cannot be stopped", job_id, exc_info=True)+ return False+ try:+ await host.kill(job_id, reason=reason)+ except KeyError:+ return True # the record is gone+ except Exception:+ # The status below still decides: a job that already ended needs no kill.+ logger.warning("Could not stop job %s (%s)", job_id, reason, exc_info=True)+ try:+ return host.status(job_id).state != "running"+ except KeyError:+ return True+ except Exception:+ logger.warning("Could not confirm that job %s stopped", job_id, exc_info=True)+ return False+++def _job_still_owned(row: Mapping[str, Any]) -> bool:+ """Whether the Watch of a ``list_job_watch_targets`` row still owns its job.++ Enabled and not removed: a job Watch is never re-enabled once it lets go+ (``watch_metadata_after_resume`` refuses), so this cannot flip back.+ """++ return bool(row["enabled"] and not row["deleted"])+++def _unowned_job_stop_reason(row: Mapping[str, Any]) -> str:+ if row["deleted"]:+ return JOB_STOP_SESSION_ARCHIVED if row["session_archived"] else JOB_STOP_WATCH_REMOVED+ return JOB_STOP_WATCH_DISABLED+++class _JobStillRunning(RuntimeError):+ """A job whose outcome was asked for has not ended."""+++@dataclass(frozen=True)+class _JobOutcome:+ #: The follow-up for the agent, in the ``bash`` tool's English.+ report: str+ exit_code: Optional[int]+ #: The Watch's ``last_error``, which people read: an i18n key and its arguments.+ error_key: Optional[str]+ error_args: dict[str, Any]+ timed_out: bool+ event_detected: bool = True+++def _job_outcome(watch: ManagedWatch, target: Mapping[str, Any]) -> _JobOutcome:+ """How the job ended, as the Watch follow-up states it (C-7 section 5).++ The command, the exit code, the elapsed time, and the output tail under the+ ``bash`` tool's rules: its truncation notices, and ``Full output:`` only for a log+ that was never bounded on disk (J4). Raises ``KeyError`` when the job record is+ gone.+ """++ from core.agent_core.tools.output import JobOutput++ host = _job_host(target)+ job_id = target["job_id"]+ meta = host.meta(job_id)+ status = host.status(job_id)+ if status.state == "running":+ raise _JobStillRunning(f"job {job_id} is still running")+ output = JobOutput(host, job_id)+ output.finish()+ text, truncation = output.render("(no output)")+ line, error_key, error_args, timed_out = _job_status(+ watch, status, host.stop_reason(job_id), meta.get("timeout_s")+ )+ elapsed = _job_elapsed_seconds(meta, str(target["state_dir"]))+ header = f"Watch {watch.id} finished" + (f" after {_format_elapsed(elapsed)}." if elapsed is not None else ".")+ parts = [header, f"Command: {target['command']}", "", text]+ if truncation is None:+ parts += ["", output.where()]+ parts += ["", line]+ return _JobOutcome(+ report="\n".join(parts),+ exit_code=status.exit_code if status.state == "exited" else None,+ error_key=error_key,+ error_args=error_args,+ timed_out=timed_out,+ )+++def _missing_job_outcome(watch: ManagedWatch, target: Mapping[str, Any]) -> _JobOutcome:+ line = f"The job record of this command is gone ({target['job_id']}), so how it ended is unknown"+ return _JobOutcome(+ report=f"Watch {watch.id} finished.\nCommand: {target['command']}\n\n{line}",+ exit_code=None,+ error_key="harness.watch.jobRecordGone",+ error_args={"job_id": target["job_id"]},+ timed_out=False,+ event_detected=False,+ )+++def _unreadable_job_outcome(watch: ManagedWatch, target: Mapping[str, Any]) -> _JobOutcome:+ line = f"The job state of this command could not be read ({target['job_id']}), so how it ended is unknown"+ return _JobOutcome(+ report=f"Watch {watch.id} finished.\nCommand: {target['command']}\n\n{line}",+ exit_code=None,+ error_key="harness.watch.jobStateUnreadable",+ error_args={"job_id": target["job_id"]},+ timed_out=False,+ event_detected=False,+ )+++def _job_status(+ watch: ManagedWatch,+ status: Any,+ stop_reason: Optional[str],+ timeout_s: Any,+) -> tuple[str, Optional[str], dict[str, Any], bool]:+ """How the job ended: the report's last line, the error's i18n key and arguments, and+ whether the job's own timeout ended it.++ A report comes only from an owning Watch, so the only stoppers it can name are the+ job's deadline, ``vibe stop``, the Watch's lifetime, and the Watch itself when the+ job's state could not be read. The lines are the ``bash``+ tool's (Pi's ``bash.ts``, MIT, Copyright (c) 2025 Mario Zechner) where it has one.+ """++ from core.agent_core.tools.args import format_number+ from core.agent_core.tools.jobs import STOP_TIMEOUT++ if status.state == "exited":+ line = f"Command exited with code {status.exit_code}"+ if status.exit_code == 0:+ return line, None, {}, False+ return line, "harness.watch.jobExitedWithCode", {"code": status.exit_code}, False+ if stop_reason == STOP_TIMEOUT:+ if isinstance(timeout_s, (int, float)) and not isinstance(timeout_s, bool):+ return f"Command timed out after {format_number(timeout_s)} seconds", "harness.watch.jobTimedOut", {}, True+ return "Command timed out", "harness.watch.jobTimedOut", {}, True+ if stop_reason == JOB_STOP_VIBE_STOP:+ return "Command stopped by `vibe stop`", "harness.watch.jobStoppedByVibeStop", {}, False+ if stop_reason == JOB_STOP_WATCH_LIFETIME:+ line = (+ f"Command stopped: Watch {watch.id} reached its lifetime of "+ f"{format_number(watch.lifetime_timeout_seconds)} seconds"+ )+ return line, "harness.watch.jobStoppedAtLifetime", {}, False+ if stop_reason == JOB_STOP_UNREADABLE:+ return "Command stopped: its job state could not be read", "harness.watch.jobStoppedUnreadable", {}, False+ return "Command terminated without an exit code", "harness.watch.jobEndedWithoutExitCode", {}, False+++def _job_elapsed_seconds(meta: Mapping[str, Any], state_dir: str) -> Optional[float]:+ """From the job's start to when it ended: its ``exit`` (or ``stopped``) record, else now."""++ created_at = meta.get("created_at")+ if not isinstance(created_at, str):+ return None+ # ``Z`` spelled out: ``fromisoformat`` accepts it only from Python 3.11.+ started = _parse_utc_timestamp(created_at.replace("Z", "+00:00"))+ if started is None:+ return None+ ended = time.time()+ for name in ("exit", "stopped"):+ try:+ ended = os.path.getmtime(os.path.join(state_dir, name))+ break+ except OSError:+ continue+ return max(0.0, ended - started.timestamp())+++def _format_elapsed(seconds: float) -> str:+ total = int(round(seconds))+ hours, rest = divmod(total, 3600)+ minutes, secs = divmod(rest, 60)+ if hours:+ return f"{hours}h {minutes}m"+ if minutes:+ return f"{minutes}m {secs}s"+ return f"{secs}s"+++async def hand_over_job(+ job: Mapping[str, Any],+ *,+ store: Optional[ManagedWatchStore] = None,+ **route: Any,+) -> str:+ """``LocalJobHost``'s ``on_hand_over``: the id of the job's Watch, adopted or created (J6).++ ``route`` goes to ``ManagedWatchStore.adopt_or_create_job_watch``: the Agent, the+ delivery target, a message, and the user context. Raises only when no Watch owns+ the job.+ """++ return await asyncio.to_thread(+ (store or ManagedWatchStore()).adopt_or_create_job_watch,+ job,+ **route,+ )+++def stop_all_jobs(store: Optional[ManagedWatchStore] = None, *, jobs_dir: Optional[Path] = None) -> None:+ """``vibe stop`` ends every command a job runs, as it ends other tool commands (plan section 5.3).++ Called holding the free service lock, so no service owns the jobs meanwhile.+ Watch-owned jobs go first, each stopped with the reason that applies: an enabled+ Watch stays enabled, and the next start re-attaches, finds the job stopped by+ ``vibe stop``, and reports that once; jobs of Watches that no longer own them are+ settled as the service would. Then every job still running in the jobs directory+ is killed the same way, a foreground ``bash`` call's included. Restarts and+ upgrades never come here, so their jobs keep running.++ ``store`` defaults to the state database's, when there is one. The Watch half and+ the directory half are independent: Watch rows that cannot be read, or a row+ that cannot be settled, never keep the directory's jobs running.+ """++ rows: list[dict[str, Any]] = []+ try:+ if store is None and paths.get_sqlite_state_path().exists():+ store = ManagedWatchStore()+ if store is not None:+ rows = store.list_job_watch_targets()+ except Exception:+ logger.warning("vibe stop could not read job Watches; it still stops every job in the jobs directory", exc_info=True)+ directories = {os.path.realpath(str(jobs_dir or agent_jobs_dir()))}+ for row in rows:+ state_dir = str(row["target"].get("state_dir") or "")+ if os.path.isabs(state_dir):+ directories.add(os.path.realpath(os.path.dirname(os.path.normpath(state_dir))))++ async def _stop_all() -> None:+ for row in rows:+ try:+ owned = _job_still_owned(row)+ reason = JOB_STOP_VIBE_STOP if owned else _unowned_job_stop_reason(row)+ if await _stop_job(row["target"], reason) and not owned:+ store.release_job_watch(row["id"], released_at=_utc_now_iso())+ except Exception:+ logger.warning("vibe stop could not settle job Watch %s", row["id"], exc_info=True)+ for directory in sorted(directories):+ await _kill_every_job(directory, JOB_STOP_VIBE_STOP)++ asyncio.run(_stop_all())+++async def _kill_every_job(jobs_dir: str, reason: str) -> None:+ """Kill, through the job host, every job under ``jobs_dir`` that still runs; ended jobs are left as they are."""++ from core.agent_core.tools.jobs import LocalJobHost++ host = LocalJobHost(jobs_dir)+ try:+ names = sorted(os.listdir(jobs_dir))+ except OSError:+ return+ for name in names:+ try:+ host.job_dir(name) # the host's own job-id rule+ await host.kill(name, reason=reason)+ except KeyError:+ continue # not a job, or no record left+ except Exception:+ logger.warning("Could not stop job %s at vibe stop", name, exc_info=True)++def _failure_hook_body(watch: ManagedWatch, *, exit_code: int, error_text: str) -> str:"""The body of the hook that tells the user why a watch stopped failing."""diff --git a/storage/background.py b/storage/background.pyindex c5c01dea5..cc2bede53 100644--- a/storage/background.py+++ b/storage/background.py@@ -1880,6 +1880,69 @@ WATCH_FOLLOW_UP_RUN_ID_METADATA_KEY = "watch_follow_up_run_id"WATCH_RECENT_EVENT_TIMESTAMPS_METADATA_KEY = "watch_recent_event_timestamps"WATCH_CIRCUIT_BREAKER_METADATA_KEY = "watch_circuit_breaker"WATCH_LIFETIME_STARTED_AT_METADATA_KEY = "watch_lifetime_started_at"+#: What a Watch waits on when it is not a waiter command: an Avibe Agent job handle+#: (plan C-7, target kind ``job``). Lives in ``metadata_json`` so the definition needs+#: no schema change:+#: ``{"kind": "job", "job_id": "job_…", "state_dir": "<absolute job directory>",+#: "command": "<the command as the agent wrote it>"}``, plus ``exit_code`` and+#: ``released_at`` once the Watch has settled the job. A job Watch's waiter columns+#: (``command_json``, ``shell_command``) stay empty on disk, so nothing that reads them,+#: including a release without job targets, can run the command a second time; the+#: read model shows the target's command in ``shell_command`` instead.+WATCH_TARGET_METADATA_KEY = "watch_target"+WATCH_TARGET_KIND_JOB = "job"+++def watch_job_target(metadata: Any) -> Optional[dict[str, Any]]:+ """The job target in a Watch's metadata, or ``None`` for a waiter-command Watch."""++ if not isinstance(metadata, dict):+ return None+ target = metadata.get(WATCH_TARGET_METADATA_KEY)+ if not isinstance(target, dict) or target.get("kind") != WATCH_TARGET_KIND_JOB:+ return None+ job_id = target.get("job_id")+ return target if isinstance(job_id, str) and job_id else None+++def _watch_target_sql(field: str) -> Any:+ """``watch_target.<field>`` of a definition row, read in SQL; NULL for a malformed blob."""++ return case(+ (+ func.json_valid(run_definitions.c.metadata_json) == 1,+ func.json_extract(+ run_definitions.c.metadata_json,+ f"$.{WATCH_TARGET_METADATA_KEY}.{field}",+ ),+ ),+ else_=None,+ )+++class JobWatchResumeRefused(RuntimeError):+ """A job Watch is never resumed: once paused or finished it owns no command."""++ code = "job_watch_not_resumable"++ def __init__(self) -> None:+ super().__init__("a Watch on a running command cannot be resumed once it is paused or finished")+++def _job_watch_id(conn: Any, job_id: Any) -> Optional[str]:+ """The one lookup of a job's Watch (J6): live and removed rows, the oldest first."""++ if not isinstance(job_id, str) or not job_id:+ return None+ existing = conn.execute(+ select(run_definitions.c.id)+ .where(run_definitions.c.definition_type == "watch")+ .where(_watch_target_sql("kind") == WATCH_TARGET_KIND_JOB)+ .where(_watch_target_sql("job_id") == job_id)+ .order_by(run_definitions.c.created_at, run_definitions.c.id)+ .limit(1)+ ).scalar_one_or_none()+ return str(existing) if existing is not None else Nonedef watch_metadata_after_resume(@@ -1889,6 +1952,12 @@ def watch_metadata_after_resume() -> dict[str, Any]:"""Start a new armed episode while retaining the repair Run admission fence."""+ if watch_job_target(metadata) is not None:+ # The one refusal point for resuming, which every resume doorway reaches before+ # it writes. A job Watch is disabled only once it no longer owns its job: a+ # pause stops the command, and a report released it. Resuming would replay a+ # released result, or erase a pause the ownership sweep has yet to act on.+ raise JobWatchResumeRefused()updated = dict(metadata)updated.pop(WATCH_RECENT_EVENT_TIMESTAMPS_METADATA_KEY, None)updated[WATCH_LIFETIME_STARTED_AT_METADATA_KEY] = resumed_at@@ -3629,6 +3698,121 @@ class SQLiteBackgroundTaskStore:expected_reference_agent_id=expected_reference_agent_id,)+ def adopt_or_create_job_watch(self, payload: dict[str, Any]) -> str:+ """The id of the Watch that owns ``payload``'s job, written from ``payload`` if none does.++ Recovery invariant J6: one job, at most one Watch, whatever crashed in between.+ Keyed by ``job_id`` across live AND removed rows -- a removed Watch still owned+ the job, and its removal is what stops it -- and decided under the write lock,+ so two hand-overs of one job converge on one row, and a Session teardown is+ either before it (seen here) or after it (its reclaim sees the row). Once this+ returns, the row is committed.+ """++ target = watch_job_target(payload.get("metadata"))+ if target is None:+ raise ValueError("a job Watch needs a job target")+ values = self._watch_values(payload)+ with self.engine.begin() as conn:+ reserve_write_lock(conn)+ existing = _job_watch_id(conn, target["job_id"])+ if existing is not None:+ return existing+ session_status = conn.execute(+ select(agent_sessions.c.status)+ .where(agent_sessions.c.id == values["session_id"])+ .limit(1)+ ).scalar_one_or_none()+ if session_status is None or session_status == "archived":+ # The Session's teardown (archive, or the ``/new`` hard delete)+ # committed first, so its reclaim could not see this Watch. It is born+ # removed, and the ownership sweep stops the job, so the command never+ # runs unowned and no follow-up targets the dead Session.+ values["deleted_at"] = _utc_now_iso()+ upsert_definition_in_connection(conn, values, expect=None, definition_type="watch")+ return str(values["id"])++ def find_job_watch(self, job_id: str) -> Optional[str]:+ """The id of the Watch that owns or owned ``job_id``, removed rows included."""++ with self.engine.connect() as conn:+ return _job_watch_id(conn, job_id)++ def list_job_watch_targets(self) -> list[dict[str, Any]]:+ """Every job Watch whose job may still need its owner: live, paused, and removed rows not yet released.++ ``session_archived`` tells a Session archive (which soft-deletes the Watch) from+ a plain removal, so the job is stopped with the reason that ended it.+ """++ session_status = (+ select(agent_sessions.c.status)+ .where(agent_sessions.c.id == run_definitions.c.session_id)+ .limit(1)+ .scalar_subquery()+ )+ stmt = (+ select(+ run_definitions.c.id,+ run_definitions.c.enabled,+ run_definitions.c.deleted_at,+ run_definitions.c.metadata_json,+ session_status.label("session_status"),+ )+ .where(run_definitions.c.definition_type == "watch")+ .where(_watch_target_sql("kind") == WATCH_TARGET_KIND_JOB)+ .where(_watch_target_sql("released_at").is_(None))+ .order_by(run_definitions.c.created_at, run_definitions.c.id)+ )+ targets: list[dict[str, Any]] = []+ with self.engine.connect() as conn:+ for row in conn.execute(stmt).mappings():+ target = watch_job_target(_json_loads(row["metadata_json"], {}))+ if target is None:+ continue+ targets.append(+ {+ "id": row["id"],+ "enabled": bool(row["enabled"]),+ "deleted": row["deleted_at"] is not None,+ "session_archived": row["session_status"] == "archived",+ "target": target,+ }+ )+ return targets++ def job_watch_settled(self, job_id: str) -> bool:+ """Whether every Watch of ``job_id`` has settled it (J5): ``True`` when none ever owned it."""++ with self.engine.connect() as conn:+ unsettled = conn.execute(+ select(run_definitions.c.id)+ .where(run_definitions.c.definition_type == "watch")+ .where(_watch_target_sql("kind") == WATCH_TARGET_KIND_JOB)+ .where(_watch_target_sql("job_id") == job_id)+ .where(_watch_target_sql("released_at").is_(None))+ .limit(1)+ ).first()+ return unsettled is None++ def release_job_watch(self, watch_id: str, *, released_at: str) -> bool:+ """Record that ``watch_id`` no longer has a job to manage; one key, whatever else the row holds."""++ with self.engine.begin() as conn:+ result = conn.execute(+ update(run_definitions)+ .where(run_definitions.c.id == watch_id)+ .where(_watch_target_sql("kind") == WATCH_TARGET_KIND_JOB)+ .values(+ metadata_json=func.json_set(+ run_definitions.c.metadata_json,+ f"$.{WATCH_TARGET_METADATA_KEY}.released_at",+ released_at,+ )+ )+ )+ return bool(result.rowcount)+def _upsert_definition(self,values: dict[str, Any],@@ -4755,6 +4939,8 @@ class SQLiteBackgroundTaskStore:[run_definitions.c.command_json,run_definitions.c.shell_command,+ # A job Watch's command lives in its target, not the waiter columns.+ _watch_target_sql("command"),run_definitions.c.prefix,run_definitions.c.cwd,]@@ -8693,6 +8879,8 @@ class SQLiteBackgroundTaskStore:}def _watch_values(self, payload: dict[str, Any]) -> dict[str, Any]:+ # A job Watch has no waiter: its command is the job's, kept in the target only.+ job_target = watch_job_target(payload.get("metadata"))return {"id": payload["id"],"definition_type": "watch",@@ -8708,14 +8896,14 @@ class SQLiteBackgroundTaskStore:"cron": None,"run_at": None,"timezone": None,- "command_json": _json_dumps(payload.get("command") or []),- "shell_command": payload.get("shell_command"),+ "command_json": _json_dumps([] if job_target else payload.get("command") or []),+ "shell_command": None if job_target else payload.get("shell_command"),"prefix": payload.get("prefix"),"cwd": payload.get("cwd"),"mode": payload.get("mode") or "once","timeout_seconds": float(payload.get("timeout_seconds", 21600.0)),"lifetime_timeout_seconds": float(payload.get("lifetime_timeout_seconds", 0.0)),- "retry_exit_codes_json": _json_dumps(payload.get("retry_exit_codes") or []),+ "retry_exit_codes_json": _json_dumps([] if job_target else payload.get("retry_exit_codes") or []),"retry_delay_seconds": float(payload.get("retry_delay_seconds", 30.0)),"post_to": payload.get("post_to"),"deliver_key": payload.get("deliver_key"),@@ -8827,6 +9015,8 @@ class SQLiteBackgroundTaskStore:@staticmethoddef _watch_from_row(row: Any) -> dict[str, Any]:+ metadata = _json_loads(row["metadata_json"], {})+ job_target = watch_job_target(metadata)return {"id": row["id"],"name": row["name"],@@ -8834,8 +9024,9 @@ class SQLiteBackgroundTaskStore:"session_policy": row["session_policy"],"session_key": row["legacy_session_key"] or "","session_id": row["session_id"],- "command": _json_loads(row["command_json"], []),- "shell_command": row["shell_command"],+ # A job Watch reads as the command it waits on, on every surface.+ "command": [] if job_target else _json_loads(row["command_json"], []),+ "shell_command": (str(job_target.get("command") or "") or None) if job_target else row["shell_command"],"prefix": row["prefix"],"message": row["message"] or row["prefix"],"message_payload": _json_loads(row["message_payload_json"], None),@@ -8865,7 +9056,7 @@ class SQLiteBackgroundTaskStore:"last_event_at": row["last_event_at"],"last_error": row["last_error"],"last_exit_code": row["last_exit_code"],- "metadata": _json_loads(row["metadata_json"], {}),+ "metadata": metadata,"lifecycle_state": _row_lifecycle_state(row),}diff --git a/tests/test_watch_job_target.py b/tests/test_watch_job_target.pynew file mode 100644index 000000000..fe53d6675--- /dev/null+++ b/tests/test_watch_job_target.py@@ -0,0 +1,954 @@+"""Watch target kind ``job`` (plan C-7 section 5.3, ``recovery.md`` J3 and J6).++Every job here is a real ``LocalJobHost`` job in a temporary directory; the Watch+service builds its own host over the same directory, as it does after a restart.+Stores default to SQLite under the per-test ``AVIBE_HOME`` (``tests/conftest.py``).+"""++from __future__ import annotations++import asyncio+import fcntl+import json+import os+import sqlite3+import sys+import time+from contextlib import closing+from datetime import datetime, timedelta, timezone+from functools import partial+from pathlib import Path+from types import SimpleNamespace++import psutil+import pytest+from sqlalchemy import insert++from config import paths+from core import watches as watches_module+from core.agent_core.tools.jobs import LocalJobHost+from core.scheduled_tasks import TaskExecutionStore+from core.watches import (+ JobWatchUpdateRefused,+ ManagedWatchService,+ ManagedWatchStore,+ WatchRuntimeStateStore,+ hand_over_job,+ stop_all_jobs,+)+from storage.background import SQLiteBackgroundTaskStore+from storage.models import agent_sessions+from storage.session_reclaim import reclaim_bound_definitions+from vibe import cli+from vibe.i18n import t as i18n_t+from vibe.ui_server import app as ui_app++from tests.ui_server_test_helpers import csrf_headers++SESSION_ID = "ses_job_watch"++[email protected](autouse=True)+def _fast_watch_loop(monkeypatch):+ monkeypatch.setattr(watches_module, "WATCH_RECONCILE_INTERVAL_SECONDS", 0.05)+ monkeypatch.setattr(watches_module, "WATCH_JOB_POLL_SECONDS", 0.05)++[email protected](autouse=True)+def _job_session():+ """The live Session the jobs run in: a hand-over binds its Watch to it."""++ now = datetime.now(timezone.utc).isoformat()+ with SQLiteBackgroundTaskStore().engine.begin() as conn:+ conn.execute(+ insert(agent_sessions).values(+ id=SESSION_ID,+ agent_backend="avibe",+ agent_variant="default",+ session_anchor=SESSION_ID,+ native_session_id=SESSION_ID,+ status="active",+ visibility="foreground",+ pinned=0,+ agent_status="idle",+ metadata_json="{}",+ created_at=now,+ updated_at=now,+ )+ )+++def _jobs_dir(tmp_path: Path) -> str:+ return str(tmp_path / "jobs")+++def _host(tmp_path: Path, store: ManagedWatchStore) -> LocalJobHost:+ """The host the adapter runs: its hand-over is Watch's adopt-or-create."""++ return LocalJobHost(_jobs_dir(tmp_path), on_hand_over=partial(hand_over_job, store=store))+++async def _start_job(host: LocalJobHost, cwd: Path, command: str, *, timeout_s=None, tool_call_id="toolu_1") -> str:+ return await host.start(+ command,+ cwd=str(cwd),+ env={"PATH": os.environ.get("PATH", "/usr/bin:/bin")},+ timeout_s=timeout_s,+ session_id=SESSION_ID,+ tool_call_id=tool_call_id,+ )+++def _service(store: ManagedWatchStore) -> ManagedWatchService:+ return ManagedWatchService(+ controller=SimpleNamespace(),+ store=store,+ request_store=TaskExecutionStore(),+ runtime_store=WatchRuntimeStateStore(),+ )+++async def _start_service(service: ManagedWatchService) -> None:+ service.start()+ if service._startup_task is not None:+ await service._startup_task+++async def _until(predicate, message: str, timeout: float = 15.0) -> None:+ deadline = time.monotonic() + timeout+ while not predicate():+ assert time.monotonic() < deadline, message+ await asyncio.sleep(0.02)+++def _follow_ups(watch_id: str) -> list:+ return [request for request in TaskExecutionStore().list_pending() if request.task_id == watch_id]+++def _gone(pid: int) -> bool:+ try:+ return psutil.Process(pid).status() == psutil.STATUS_ZOMBIE+ except psutil.NoSuchProcess:+ return True+++def _sql(statement: str, *params) -> list[tuple]:+ with closing(sqlite3.connect(paths.get_sqlite_state_path())) as conn, conn:+ return conn.execute(statement, params).fetchall()+++def _settled_row(watch_id: str) -> dict:+ """The Watch row as the Harness shows it once its follow-up turn is done."""++ _sql(+ "UPDATE agent_runs SET status = 'succeeded', completed_at = updated_at "+ "WHERE definition_id = ? AND run_type = 'watch'",+ watch_id,+ )+ return SQLiteBackgroundTaskStore().get_watch(watch_id)+++def _released(watch_id: str) -> bool:+ return all(row["id"] != watch_id for row in SQLiteBackgroundTaskStore().list_job_watch_targets())+++def test_restart_while_a_job_watch_waits_reattaches_and_reports_once(tmp_path: Path) -> None:+ release = tmp_path / "release"+ command = f"while [ ! -e {release} ]; do sleep 0.05; done; echo finished; exit 3"++ async def run() -> tuple[str, dict]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, command)+ watch_id = await host.hand_over(job_id)++ first = _service(store)+ await _start_service(first)+ await _until(lambda: first.store.get_watch(watch_id).last_started_at, "the job Watch never started waiting")+ assert not store.job_watch_settled(job_id)+ # What a crash leaves behind for the next service's stale-worker reaping.+ crashed_runtime = first._runtime_state_payload()+ await first.stop()+ WatchRuntimeStateStore().write(crashed_runtime)+ assert host.status(job_id).state == "running"++ second_store = ManagedWatchStore()+ second = _service(second_store)+ await _start_service(second)+ await asyncio.sleep(0.2)+ assert host.status(job_id).state == "running", "restart recovery touched the job"+ release.touch()+ await _until(lambda: not second_store.get_watch(watch_id).enabled, "the re-attached Watch never reported")+ await asyncio.sleep(0.2) # a second report would land here+ await second.stop()+ # J5: the job's files may go once its tool call has settled too.+ assert second_store.job_watch_settled(job_id)+ return watch_id, crashed_runtime++ watch_id, crashed_runtime = asyncio.run(run())++ # The job Watch's runtime entry names no process, so reaping has nothing to kill.+ assert crashed_runtime["watches"][watch_id]["pid"] is None+ follow_ups = _follow_ups(watch_id)+ assert len(follow_ups) == 1+ prompt = follow_ups[0].prompt+ assert f"Command: {command}" in prompt+ assert "\nfinished\n" in prompt+ assert prompt.endswith("Command exited with code 3")+ assert follow_ups[0].session_id == SESSION_ID+++def _end_by_remove(store: ManagedWatchStore, watch_id: str) -> None:+ assert store.remove_watch(watch_id)+++def _end_by_pause(store: ManagedWatchStore, watch_id: str) -> None:+ store.set_enabled(watch_id, False)+++def _archive_session() -> None:+ """The archive dialog's teardown: the Session is archived and its Watches soft-deleted."""++ with SQLiteBackgroundTaskStore().engine.begin() as conn:+ conn.execute(agent_sessions.update().where(agent_sessions.c.id == SESSION_ID).values(status="archived"))+ reclaim_bound_definitions(conn, SESSION_ID, mode="delete")+ watches_module._publish_watch_definitions_updated()+++def _end_by_archive(store: ManagedWatchStore, watch_id: str) -> None:+ _archive_session()+++def _end_by_new(store: ManagedWatchStore, watch_id: str) -> None:+ # ``/new``'s hard delete: the Session's Watches are paused and its row removed, in one transaction.+ with SQLiteBackgroundTaskStore().engine.begin() as conn:+ reclaim_bound_definitions(conn, SESSION_ID, mode="pause")+ conn.execute(agent_sessions.delete().where(agent_sessions.c.id == SESSION_ID))+ watches_module._publish_watch_definitions_updated()+++ ("end", "reason"),+ [+ (_end_by_remove, "watch_removed"),+ (_end_by_pause, "watch_disabled"),+ (_end_by_archive, "session_archived"),+ (_end_by_new, "watch_disabled"),+ ],+ ids=["remove", "pause", "archive", "new"],+)+def test_ending_ownership_kills_the_job_process_tree(tmp_path: Path, end, reason: str) -> None:+ child_pid_file = tmp_path / "child.pid"+ command = f"sleep 60 & echo $! > {child_pid_file}; wait"++ async def run() -> tuple[LocalJobHost, str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, command)+ watch_id = await host.hand_over(job_id)+ service = _service(store)+ await _start_service(service)+ await _until(lambda: child_pid_file.exists() and child_pid_file.read_text().strip(), "no background child")+ await _until(lambda: watch_id in service._active_tasks, "the job Watch never started")+ end(store, watch_id)+ await _until(lambda: host.status(job_id).state != "running", "the job outlived its Watch")+ await _until(lambda: _released(watch_id), "the Watch was never released")+ await service.stop()+ return host, job_id, watch_id++ host, job_id, watch_id = asyncio.run(run())++ assert host.stop_reason(job_id) == reason+ assert _gone(int(child_pid_file.read_text()))+ assert _follow_ups(watch_id) == []+++def _resume_in_the_cli(watch_id: str, capsys) -> tuple[str, str]:+ assert cli.cmd_watch_set_enabled(watch_id, True) == 1+ error = json.loads(capsys.readouterr().err)+ return error["code"], error["error"]+++def _resume_in_the_web_ui(watch_id: str, capsys) -> tuple[str, str]:+ client = ui_app.test_client()+ response = client.patch(+ f"/api/harness/watches/{watch_id}", json={"enabled": True}, headers=csrf_headers(client)+ )+ assert response.status_code == 409+ error = response.get_json()["error"]+ return error["code"], error["message"]++[email protected]("state", ["paused", "released"])[email protected]("resume", [_resume_in_the_cli, _resume_in_the_web_ui], ids=["cli", "web-ui"])+def test_a_job_watch_that_no_longer_owns_its_job_is_never_resumed(+ tmp_path: Path, capsys, monkeypatch, state: str, resume+) -> None:+ monkeypatch.setattr(cli, "_configured_cli_language", lambda: "zh")++ async def run() -> tuple[LocalJobHost, str, str, tuple[str, str]]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "echo done" if state == "released" else "sleep 60")+ watch_id = await host.hand_over(job_id)+ if state == "released":+ first = _service(store)+ await _start_service(first)+ await _until(lambda: not store.get_watch(watch_id).enabled, "the job never reported")+ await first.stop()+ else:+ # Paused, and resumed before any service has seen the pause.+ store.set_enabled(watch_id, False)+ refusal = resume(watch_id, capsys)+ # A repeated hand-over adopts the same Watch and does not re-arm it either.+ assert await hand_over_job(host.meta(job_id), store=ManagedWatchStore()) == watch_id+ reloaded = ManagedWatchStore()+ service = _service(reloaded)+ await _start_service(service)+ await _until(lambda: host.status(job_id).state != "running", "the paused job kept running")+ await _until(lambda: _released(watch_id), "the Watch was never released")+ await asyncio.sleep(0.2) # a replayed report would land here+ await service.stop()+ return host, job_id, watch_id, refusal++ host, job_id, watch_id, (code, message) = asyncio.run(run())++ assert code == "job_watch_not_resumable"+ language = "zh" if resume is _resume_in_the_cli else "en"+ assert message == i18n_t("error.jobWatchResumeRefused.message", language)+ assert not ManagedWatchStore().get_watch(watch_id).enabled+ if state == "released":+ assert len(_follow_ups(watch_id)) == 1+ assert host.stop_reason(job_id) is None+ else:+ assert _follow_ups(watch_id) == []+ assert host.stop_reason(job_id) == "watch_disabled"+++def test_changing_a_job_watch_waiter_is_refused_in_the_user_language(tmp_path: Path, capsys, monkeypatch) -> None:+ async def hand_over() -> tuple[LocalJobHost, str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "sleep 60")+ return host, job_id, await host.hand_over(job_id)++ host, job_id, watch_id = asyncio.run(hand_over())+ monkeypatch.setattr(cli, "_configured_cli_language", lambda: "zh")+ monkeypatch.setattr(sys, "argv", ["vibe", "watch", "update", watch_id, "--shell", "pytest -q"])+ with pytest.raises(SystemExit) as exit_info:+ cli.main()++ assert exit_info.value.code == 1+ error = json.loads(capsys.readouterr().err)+ assert error["code"] == "job_watch_update_refused"+ assert error["error"] == i18n_t("error.jobWatchUpdateRefused.message", "zh")+ assert ManagedWatchStore().get_watch(watch_id).job_target["command"] == "sleep 60"++ # What it does take goes through the same doorway.+ monkeypatch.setattr(sys, "argv", ["vibe", "watch", "update", watch_id, "--name", "slow tests"])+ with pytest.raises(SystemExit) as exit_info:+ cli.main()+ assert exit_info.value.code == 0, capsys.readouterr().err+ assert ManagedWatchStore().get_watch(watch_id).name == "slow tests"+ asyncio.run(host.kill(job_id))+++def _session_gone() -> None:+ with SQLiteBackgroundTaskStore().engine.begin() as conn:+ conn.execute(agent_sessions.delete().where(agent_sessions.c.id == SESSION_ID))+++def _update_fields(watch) -> dict:+ """Every field ``update_watch`` takes, as the Watch has them now."""++ return {+ "name": watch.name,+ "session_key": watch.session_key,+ "session_id": watch.session_id,+ "command": list(watch.command),+ "shell_command": watch.shell_command,+ "prefix": watch.prefix,+ "cwd": watch.cwd,+ "mode": watch.mode,+ "timeout_seconds": watch.timeout_seconds,+ "lifetime_timeout_seconds": watch.lifetime_timeout_seconds,+ "retry_exit_codes": list(watch.retry_exit_codes),+ "retry_delay_seconds": watch.retry_delay_seconds,+ "post_to": watch.post_to,+ "deliver_key": watch.deliver_key,+ "agent_name": watch.agent_name,+ "session_policy": watch.session_policy,+ "message": watch.message,+ "metadata": dict(watch.metadata),+ }+++_FIXED_JOB_WATCH_FIELDS = {+ "session_id": "ses_other",+ "session_key": "slack::channel::C999",+ "session_policy": "create_once",+ "agent_name": "claude",+ "post_to": "channel",+ "deliver_key": "slack::channel::C999",+ "metadata": {"binding_follows_session": True},+ "command": ["pytest", "-q"],+ "shell_command": "pytest -q",+ "cwd": "/tmp",+ "mode": "forever",+ "timeout_seconds": 60.0,+ "retry_exit_codes": [75],+ "retry_delay_seconds": 5.0,+}++[email protected](("field", "value"), sorted(_FIXED_JOB_WATCH_FIELDS.items()))+def test_a_job_watch_takes_no_update_beyond_its_name_message_and_lifetime(tmp_path: Path, field: str, value) -> None:+ async def hand_over() -> tuple[LocalJobHost, str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "sleep 60")+ return host, job_id, await host.hand_over(job_id)++ host, job_id, watch_id = asyncio.run(hand_over())+ store = ManagedWatchStore()+ before = store.get_watch(watch_id).to_dict()+ fields = _update_fields(store.get_watch(watch_id))+ if field == "metadata":+ value = {**fields["metadata"], **value}++ with pytest.raises(JobWatchUpdateRefused):+ store.update_watch(watch_id, **{**fields, field: value})+ assert ManagedWatchStore().get_watch(watch_id).to_dict() == before++ # The three it does take still land.+ allowed = {**fields, "name": "tests", "message": "Tests finished.", "lifetime_timeout_seconds": 900.0}+ updated = store.update_watch(watch_id, **allowed)+ assert (updated.name, updated.message, updated.lifetime_timeout_seconds) == ("tests", "Tests finished.", 900.0)+ assert ManagedWatchStore().get_watch(watch_id).session_id == SESSION_ID+ asyncio.run(host.kill(job_id))+++def _corrupt_deadline(meta: dict) -> dict:+ return {**meta, "deadline_at": "not a time"}+++def test_an_unreadable_job_state_stops_the_job_before_the_watch_lets_go(tmp_path: Path) -> None:+ async def run() -> tuple[LocalJobHost, str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "echo started; sleep 60")+ watch_id = await host.hand_over(job_id)+ meta_path = Path(host.job_dir(job_id)) / "meta.json"+ meta_path.write_text(json.dumps(_corrupt_deadline(json.loads(meta_path.read_text()))))+ service = _service(store)+ await _start_service(service)+ await _until(lambda: not store.get_watch(watch_id).enabled, "the Watch never settled the unreadable job")+ await service.stop()+ return host, job_id, watch_id++ host, job_id, watch_id = asyncio.run(run())++ assert host.status(job_id).state == "gone"+ assert host.stop_reason(job_id) == "watch_unreadable_state"+ (follow_up,) = _follow_ups(watch_id)+ assert "started" in follow_up.prompt+ assert follow_up.prompt.endswith("Command stopped: its job state could not be read")+ assert SQLiteBackgroundTaskStore().get_watch(watch_id)["last_error"] == i18n_t(+ "harness.watch.jobStoppedUnreadable", "en"+ )+ assert _released(watch_id)++[email protected]("owned", [True, False], ids=["owning-watch", "removed-watch"])+def test_a_job_that_cannot_be_stopped_is_never_released(tmp_path: Path, owned: bool) -> None:+ async def run() -> tuple[LocalJobHost, str, str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "sleep 60")+ watch_id = await host.hand_over(job_id)+ meta_path = Path(host.job_dir(job_id)) / "meta.json"+ intact = meta_path.read_text()+ # Unreadable: neither its deadline nor the process identity a kill needs can be read.+ meta_path.write_text("{ not json")+ if not owned:+ store.remove_watch(watch_id)+ service = _service(store)+ await _start_service(service)+ if owned:+ await _until(lambda: store.get_watch(watch_id).last_error, "the Watch never said it was stuck")+ else:+ await asyncio.sleep(0.5) # several sweeps+ await service.stop()+ meta_path.write_text(intact)+ return host, job_id, watch_id, intact++ host, job_id, watch_id, _intact = asyncio.run(run())++ assert host.status(job_id).state == "running"+ assert not _released(watch_id)+ assert _follow_ups(watch_id) == []+ if owned:+ row = SQLiteBackgroundTaskStore().get_watch(watch_id)+ assert row["enabled"]+ assert row["last_error"] == i18n_t("harness.watch.jobUnmanageable", "en")+ asyncio.run(host.kill(job_id))+++ ("teardown", "reason"),+ [(_archive_session, "session_archived"), (_session_gone, "watch_removed")],+ ids=["archived", "deleted"],+)+def test_a_hand_over_after_its_session_teardown_still_ends_the_job(tmp_path: Path, teardown, reason: str) -> None:+ async def run() -> tuple[LocalJobHost, str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "sleep 60")+ # The teardown's reclaim commits before the hand-over, so it cannot see this Watch.+ teardown()+ watch_id = await host.hand_over(job_id)+ service = _service(store)+ await _start_service(service)+ await _until(lambda: host.status(job_id).state != "running", "the job outlived its Session")+ await _until(lambda: _released(watch_id), "the Watch was never released")+ await service.stop()+ return host, job_id, watch_id++ host, job_id, watch_id = asyncio.run(run())++ assert host.stop_reason(job_id) == reason+ assert _follow_ups(watch_id) == []+++def test_watch_lifetime_kills_the_job_and_reports_it(tmp_path: Path) -> None:+ async def run() -> tuple[LocalJobHost, str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "echo started; sleep 60")+ watch_id = await host.hand_over(job_id)+ watch = store.get_watch(watch_id)+ watch.lifetime_timeout_seconds = 0.5+ store.upsert_watch(watch)+ service = _service(store)+ await _start_service(service)+ await _until(lambda: not store.get_watch(watch_id).enabled, "the lifetime never ended the Watch")+ await service.stop()+ return host, job_id, watch_id++ host, job_id, watch_id = asyncio.run(run())++ assert host.status(job_id).state == "gone"+ assert host.stop_reason(job_id) == "watch_lifetime"+ (follow_up,) = _follow_ups(watch_id)+ assert "started" in follow_up.prompt+ assert follow_up.prompt.endswith(f"Command stopped: Watch {watch_id} reached its lifetime of 0.5 seconds")+ assert _released(watch_id)+++def test_job_deadline_holds_while_the_watch_owns_it(tmp_path: Path) -> None:+ async def run() -> tuple[LocalJobHost, str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "sleep 60", timeout_s=60)+ # Move the recorded deadline in, past the one the wrapper enforces from its argv,+ # so only the job's owner can enforce it in time.+ meta_path = Path(host.job_dir(job_id)) / "meta.json"+ meta = json.loads(meta_path.read_text())+ deadline = datetime.now(timezone.utc) + timedelta(seconds=0.5)+ meta["deadline_at"] = deadline.isoformat(timespec="milliseconds").replace("+00:00", "Z")+ meta["timeout_s"] = 0.5+ meta_path.write_text(json.dumps(meta))+ watch_id = await host.hand_over(job_id)+ service = _service(store)+ await _start_service(service)+ await _until(lambda: not store.get_watch(watch_id).enabled, "the deadline never ended the job", timeout=10)+ await service.stop()+ return host, job_id, watch_id++ host, job_id, watch_id = asyncio.run(run())++ assert host.stop_reason(job_id) == "timeout"+ (follow_up,) = _follow_ups(watch_id)+ assert follow_up.prompt.endswith("Command timed out after 0.5 seconds")+ row = _settled_row(watch_id)+ assert row["lifecycle_detail"] == "timeout"+ assert row["metadata"]["last_command_timed_out"] is True+++def test_hand_over_returns_the_watch_that_already_owns_the_job(tmp_path: Path) -> None:+ async def run() -> None:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "sleep 60")+ watch_id = await host.hand_over(job_id)++ # A crash before meta.json recorded the hand-over: the job host asks again.+ meta_path = Path(host.job_dir(job_id)) / "meta.json"+ meta = json.loads(meta_path.read_text())+ meta["watch_id"] = None+ meta_path.write_text(json.dumps(meta))+ assert await host.hand_over(job_id) == watch_id+ # A removed Watch still owned the job: no second Watch replaces it.+ store.remove_watch(watch_id)+ assert await hand_over_job(host.meta(job_id), store=ManagedWatchStore()) == watch_id++ # Two hand-overs of one job at once, from separate stores, converge on one Watch.+ other_job = await _start_job(host, tmp_path, "sleep 60", tool_call_id="toolu_2")+ other_meta = host.meta(other_job)+ ids = await asyncio.gather(+ hand_over_job(other_meta, store=ManagedWatchStore()),+ hand_over_job(other_meta, store=ManagedWatchStore()),+ )+ assert ids[0] == ids[1]+ for job in (job_id, other_job):+ await host.kill(job)++ asyncio.run(run())++ rows = _sql(+ "SELECT json_extract(metadata_json, '$.watch_target.job_id'), count(*) FROM run_definitions "+ "WHERE definition_type = 'watch' GROUP BY 1"+ )+ assert sorted(count for _job, count in rows) == [1, 1]+++ "retry",+ [+ {"user_context": {}}, # the caller has lost Editor access since the first hand-over+ {"agent_name": "no-such-agent", "user_context": {}},+ {"drop": "command"}, # a job record that could no longer create a Watch+ ],+ ids=["access-revoked", "agent-gone", "record-incomplete"],+)+def test_a_repeated_hand_over_returns_the_owner_whatever_creation_would_now_refuse(tmp_path: Path, retry: dict) -> None:+ async def run() -> tuple[LocalJobHost, str, str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "sleep 60")+ # The first hand-over committed its Watch; a crash lost meta.json's record of it.+ watch_id = await hand_over_job(host.meta(job_id), store=store)+ meta = host.meta(job_id)+ if "drop" in retry:+ meta.pop(retry["drop"])+ route = {key: value for key, value in retry.items() if key != "drop"}+ again = await hand_over_job(meta, store=ManagedWatchStore(), **route)+ await host.kill(job_id)+ return host, job_id, watch_id, again++ _host_, _job, watch_id, again = asyncio.run(run())++ assert again == watch_id+++def test_a_hand_over_that_loses_its_creation_check_to_a_concurrent_owner_returns_that_owner(+ tmp_path: Path, monkeypatch+) -> None:+ from storage import resource_access_service++ async def run() -> tuple[str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "sleep 60")+ meta = host.meta(job_id)+ concurrent: list[str] = []+ original = resource_access_service.ensure_harness_definition_write++ def lose_the_race(user_context=None):+ # Another hand-over of this job commits its Watch after this one looked, and+ # this one's access check then fails.+ monkeypatch.setattr(resource_access_service, "ensure_harness_definition_write", original)+ concurrent.append(ManagedWatchStore().adopt_or_create_job_watch(meta))+ raise resource_access_service.ResourceAccessError(resource_access_service.HARNESS_ACCESS_FORBIDDEN_CODE)++ monkeypatch.setattr(resource_access_service, "ensure_harness_definition_write", lose_the_race)+ watch_id = await hand_over_job(meta, store=store)+ await host.kill(job_id)+ return watch_id, concurrent[0]++ watch_id, concurrent_id = asyncio.run(run())++ assert watch_id == concurrent_id+++ ("exit_code", "detail"),+ [(0, "normal"), (64, "error"), (75, "error"), (124, "error")],+)+def test_job_exit_code_is_reported_never_read_as_a_watch_control_code(+ tmp_path: Path, exit_code: int, detail: str+) -> None:+ async def run() -> str:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, f"echo 'avibe-watch: no-event'; exit {exit_code}")+ await host.wait(job_id, deadline_s=10)+ watch_id = await host.hand_over(job_id)+ service = _service(store)+ await _start_service(service)+ await _until(lambda: not store.get_watch(watch_id).enabled, "the Watch never reported the exit")+ await asyncio.sleep(0.2) # a retry would start another cycle here+ await service.stop()+ return watch_id++ watch_id = asyncio.run(run())++ # 64 with the no-event marker, 75, and 124 mean something to a waiter; here they are the command's.+ (follow_up,) = _follow_ups(watch_id)+ assert follow_up.prompt.endswith(f"Command exited with code {exit_code}")+ row = _settled_row(watch_id)+ assert row["lifecycle_state"] == "finished"+ assert row["lifecycle_detail"] == detail+ assert row["last_exit_code"] is None+ assert row["metadata"]["watch_target"]["exit_code"] == exit_code+++ ("command", "kept", "dropped", "notice"),+ [+ ("seq 1 3000", "\n1001\n", "\n1000\n", "[Showing lines 1001-3000 of 3000. "),+ ("head -c 4000000 /dev/zero | tr '\\0' 'x'; echo", "xxxx\n", None, "tail.log were omitted"),+ ],+ ids=["long-output", "log-bounded-on-disk"],+)+def test_follow_up_states_the_tail_under_the_bash_rules(+ tmp_path: Path, command: str, kept: str, dropped, notice: str+) -> None:+ async def run() -> tuple[str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, command)+ watch_id = await host.hand_over(job_id)+ service = _service(store)+ await _start_service(service)+ await _until(lambda: not store.get_watch(watch_id).enabled, "the Watch never reported")+ await service.stop()+ return watch_id, host.output_path(job_id)++ watch_id, output_path = asyncio.run(run())++ (follow_up,) = _follow_ups(watch_id)+ prompt = follow_up.prompt+ assert prompt.startswith(f"Watch {watch_id} finished after ")+ assert f"Command: {command}\n" in prompt+ assert kept in prompt and (dropped is None or dropped not in prompt)+ assert notice in prompt and output_path in prompt+ # J4: a log bounded on disk is never promised as the full output.+ assert dropped is not None or "Full output:" not in prompt+ assert prompt.endswith("Command exited with code 0")+++def test_a_job_watch_reads_as_its_command_and_stores_no_waiter(tmp_path: Path) -> None:+ command = "pytest -q tests/test_slow.py"++ async def run() -> str:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, f"sleep 60 # {command}")+ watch_id = await hand_over_job({**host.meta(job_id), "command": command}, store=store)+ await host.kill(job_id)+ return watch_id++ watch_id = asyncio.run(run())++ sqlite = SQLiteBackgroundTaskStore()+ shown = cli._watch_projection_payload(sqlite.get_watch(watch_id))+ listed = sqlite.list_watches_page(page_request=None, query="test_slow").items+ assert shown["command_preview"] == command+ assert shown["display_name"] == command+ assert shown["shell_command"] == command+ assert [row["id"] for row in listed] == [watch_id]+ # Nothing a reader without job targets could run.+ assert _sql("SELECT command_json, shell_command FROM run_definitions WHERE id = ?", watch_id) == [("[]", None)]+++_RELEASED_COMMAND_WATCHES = [+ pytest.param(+ {+ "id": "a1b2c3d4e5f6",+ "name": "Wait for CI",+ "session_key": "slack::channel::C123",+ "session_id": "sesk8m4q2p7x",+ "agent_name": "claude",+ "session_policy": "existing",+ "command": ["python3", "wait_pr.py", "--forever"],+ "shell_command": None,+ "prefix": "CI finished.",+ "message": "CI finished.",+ "cwd": "/work/repo",+ "mode": "forever",+ "timeout_seconds": 0.0,+ "lifetime_timeout_seconds": 0.0,+ "retry_exit_codes": [75],+ "retry_delay_seconds": 30.0,+ "post_to": None,+ "deliver_key": None,+ "enabled": True,+ "metadata": {"watch_lifetime_started_at": "2026-05-15T00:00:00+00:00", "delivered_reports": 2},+ },+ id="argv-forever",+ ),+ pytest.param(+ {+ "id": "f6e5d4c3b2a1",+ "name": None,+ "session_key": "",+ "session_id": "sesk8m4q2p7x",+ "command": [],+ "shell_command": "gh run watch 123 --exit-status",+ "mode": "once",+ "timeout_seconds": 600.0,+ "retry_exit_codes": [75, 124],+ "enabled": False,+ "last_exit_code": 64,+ "metadata": {},+ },+ id="shell-once",+ ),+ pytest.param(+ {"id": "0a1b2c3d4e5f", "session_key": "slack::channel::C9", "command": ["true"]},+ id="legacy-minimal",+ ),+]++[email protected]("released", _RELEASED_COMMAND_WATCHES)+def test_released_command_watch_shapes_load_unchanged(tmp_path: Path, released: dict) -> None:+ """Rows and ``watches.json`` entries written before job targets keep their waiter."""++ legacy = ManagedWatchStore(tmp_path / "watches.json")+ (tmp_path / "watches.json").write_text(json.dumps({"watches": [released]}))+ legacy.load()+ from_file = legacy.get_watch(released["id"])++ sqlite = SQLiteBackgroundTaskStore()+ stamps = {"created_at": "2026-05-15T00:00:00+00:00", "updated_at": "2026-05-15T00:00:00+00:00"}+ assert sqlite.upsert_watch({**stamps, **released})+ from_row = ManagedWatchStore().get_watch(released["id"])++ for watch in (from_file, from_row):+ assert watch.job_target is None+ assert watch.command == released["command"]+ assert watch.shell_command == released.get("shell_command")+ assert watch.retry_exit_codes == released.get("retry_exit_codes", [75])+ assert watch.mode == released.get("mode", "once")+ # Writing the loaded Watch back stores the same waiter it was loaded with.+ assert sqlite.upsert_watch(from_row.to_dict())+ stored = _sql(+ "SELECT command_json, shell_command, retry_exit_codes_json FROM run_definitions WHERE id = ?",+ released["id"],+ )+ assert stored == [+ (+ json.dumps(released["command"], separators=(",", ":")),+ released.get("shell_command"),+ json.dumps(released.get("retry_exit_codes", [75]), separators=(",", ":")),+ )+ ]+++def test_vibe_stop_ends_owned_jobs_and_the_watch_reports_after_the_next_start(tmp_path: Path, monkeypatch) -> None:+ async def hand_over() -> tuple[LocalJobHost, str, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "sleep 60")+ return host, job_id, await host.hand_over(job_id)++ host, job_id, watch_id = asyncio.run(hand_over())+ monkeypatch.setattr(cli, "_pid_file_points_to_live_process", lambda path: False)+ monkeypatch.setattr(cli.runtime, "stop_service", lambda **kwargs: False)+ monkeypatch.setattr(cli.runtime, "stop_ui", lambda **kwargs: False)+ monkeypatch.setattr(cli, "_stop_opencode_server", lambda *args: False)+ monkeypatch.setattr(cli, "_write_status", lambda state, detail=None: None)++ assert cli.cmd_stop() == 0++ assert host.status(job_id).state == "gone"+ assert host.stop_reason(job_id) == "vibe_stop"+ assert ManagedWatchStore().get_watch(watch_id).enabled++ async def start_again() -> None:+ store = ManagedWatchStore()+ service = _service(store)+ await _start_service(service)+ await _until(lambda: not store.get_watch(watch_id).enabled, "the Watch never reported the stop")+ await service.stop()++ asyncio.run(start_again())+ (follow_up,) = _follow_ups(watch_id)+ assert follow_up.prompt.endswith("Command stopped by `vibe stop`")+ # Called again, there is nothing left to stop and nothing to report twice.+ stop_all_jobs(ManagedWatchStore())+ assert len(_follow_ups(watch_id)) == 1+++def test_vibe_stop_ends_a_foreground_job_that_no_watch_owns(tmp_path: Path, monkeypatch) -> None:+ # The adapter's host: the bash tool's jobs live in the state's jobs directory.+ host = LocalJobHost(str(watches_module.agent_jobs_dir()))+ job_id = asyncio.run(_start_job(host, tmp_path, "sleep 60 & wait"))+ assert host.meta(job_id)["watch_id"] is None # still in the foreground, never handed over+ monkeypatch.setattr(cli, "_pid_file_points_to_live_process", lambda path: False)+ monkeypatch.setattr(cli.runtime, "stop_service", lambda **kwargs: False)+ monkeypatch.setattr(cli.runtime, "stop_ui", lambda **kwargs: False)+ monkeypatch.setattr(cli, "_stop_opencode_server", lambda *args: False)+ monkeypatch.setattr(cli, "_write_status", lambda state, detail=None: None)++ assert cli.cmd_stop() == 0++ assert host.status(job_id).state == "gone"+ assert host.stop_reason(job_id) == "vibe_stop"+++def _corrupt_state_database() -> None:+ db = paths.get_sqlite_state_path()+ for suffix in ("-wal", "-shm"):+ Path(f"{db}{suffix}").unlink(missing_ok=True)+ db.write_bytes(b"not a database" * 64)+++def _drop_definitions_table() -> None:+ _sql("DROP TABLE run_definitions")++[email protected]("break_store", [_corrupt_state_database, _drop_definitions_table], ids=["corrupt-db", "no-table"])+def test_vibe_stop_ends_foreground_jobs_even_when_the_watch_store_cannot_be_read(+ tmp_path: Path, monkeypatch, break_store+) -> None:+ host = LocalJobHost(str(watches_module.agent_jobs_dir()))+ job_id = asyncio.run(_start_job(host, tmp_path, "sleep 60"))+ break_store()+ monkeypatch.setattr(cli, "_pid_file_points_to_live_process", lambda path: False)+ monkeypatch.setattr(cli.runtime, "stop_service", lambda **kwargs: False)+ monkeypatch.setattr(cli.runtime, "stop_ui", lambda **kwargs: False)+ monkeypatch.setattr(cli, "_stop_opencode_server", lambda *args: False)+ monkeypatch.setattr(cli, "_write_status", lambda state, detail=None: None)++ assert cli.cmd_stop() == 0++ assert host.status(job_id).state == "gone"+ assert host.stop_reason(job_id) == "vibe_stop"+++def test_vibe_stop_leaves_the_jobs_of_a_service_that_holds_the_lock(tmp_path: Path, monkeypatch) -> None:+ async def hand_over() -> tuple[LocalJobHost, str]:+ store = ManagedWatchStore()+ host = _host(tmp_path, store)+ job_id = await _start_job(host, tmp_path, "sleep 60")+ await host.hand_over(job_id)+ return host, job_id++ host, job_id = asyncio.run(hand_over())+ monkeypatch.setattr(cli, "_pid_file_points_to_live_process", lambda path: False)+ monkeypatch.setattr(cli.runtime, "stop_service", lambda **kwargs: False)+ monkeypatch.setattr(cli.runtime, "stop_ui", lambda **kwargs: False)+ monkeypatch.setattr(cli, "_stop_opencode_server", lambda *args: False)+ monkeypatch.setattr(cli, "_write_status", lambda state, detail=None: None)+ lock_path = cli.runtime.get_service_lock_path()+ lock_path.parent.mkdir(parents=True, exist_ok=True)+ # A service that started after this stop's own stop of the service owns the jobs now.+ with lock_path.open("a+") as held:+ fcntl.flock(held.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)+ assert cli.cmd_stop() == 0++ assert host.status(job_id).state == "running"+ asyncio.run(host.kill(job_id))diff --git a/vibe/cli.py b/vibe/cli.pyindex 843bd3cd9..f3d6bafa7 100644--- a/vibe/cli.py+++ b/vibe/cli.py@@ -63,6 +63,7 @@ from core.watches import (NO_EVENT_EXIT_CODE,WATCH_RECOVERY_ENTRY_TIMEOUT_SECONDS,WATCH_RECONCILE_INTERVAL_SECONDS,+ JobWatchUpdateRefused,ManagedWatchStore,WatchRuntimeStateStore,)@@ -112,6 +113,7 @@ from storage.background import (DefinitionWriteConflict,SQLiteBackgroundTaskStore,TASK_RETIREMENT_SCHEDULE_MISSED,+ JobWatchResumeRefused,TaskResumeBlocked,TaskScheduleRetired,compute_next_run_at,@@ -10955,6 +10957,18 @@ def cmd_watch_set_enabled(watch_id: str, enabled: bool):return 1try:updated = store.set_enabled(watch_id, enabled)+ except JobWatchResumeRefused as exc:+ lang = _configured_cli_language()+ _print_task_error(+ TaskCliError(+ i18n_t("error.jobWatchResumeRefused.message", lang),+ code=exc.code,+ hint=i18n_t("error.jobWatchResumeRefused.hint", lang, id=watch_id),+ help_command=f"vibe watch remove {watch_id}",+ details={"watch_id": watch_id},+ )+ )+ return 1except DefinitionWriteConflict as exc:_print_task_error(_definition_conflict_cli_error(@@ -11340,6 +11354,23 @@ def cmd_watch_update(args):watch_payload = _watch_mutation_payload(updated, runtime_entry)_print_definition_payload(watch_payload, warnings=warnings)return 0+ except JobWatchUpdateRefused as exc:+ if reserved_session_id:+ _release_cli_session_reservation(+ reserved_session_id,+ reason="watch update failed before its Session reservation was adopted",+ )+ lang = _configured_cli_language()+ _print_task_error(+ TaskCliError(+ i18n_t("error.jobWatchUpdateRefused.message", lang),+ code=exc.code,+ hint=i18n_t("error.jobWatchUpdateRefused.hint", lang, id=exc.watch_id),+ help_command="vibe watch update --help",+ details={"watch_id": exc.watch_id},+ )+ )+ return 1except DefinitionWriteConflict as exc:if reserved_session_id:_release_cli_session_reservation(@@ -13997,6 +14028,22 @@ def _desktop_provenance_refused(return True+def _stop_jobs() -> None:+ """End every command a job runs, Watch-owned or foreground, as a full stop ends other tool commands.++ Restarts and upgrades (and a desktop Runtime's scoped stop, which may be one)+ never come here, so those commands keep running. A failure is logged and+ leaves the commands to their own deadlines.+ """++ try:+ from core.watches import stop_all_jobs++ stop_all_jobs()+ except Exception:+ logger.warning("Could not stop the commands jobs run", exc_info=True)++def cmd_stop(*, expect_runtime_id: str | None = None):from vibe.desktop_runtime import desktop_caller_provenance@@ -14018,6 +14065,9 @@ def cmd_stop(*, expect_runtime_id: str | None = None):except runtime.DesktopRuntimeClaimRefused as refusal:_print_provenance_refusal("desktopRuntime.stopRefused", refusal)return 3+ # Holding the free service lock: neither a service that failed to stop nor one+ # started since owns these commands while they are stopped.+ runtime.desktop_service_lock_presence(while_absent=_stop_jobs)from vibe.desktop_backends import reap_abandoned_desktop_backend_installs# Each tree's owner decides whether it is abandoned, so the reap runsdiff --git a/vibe/i18n/en.json b/vibe/i18n/en.jsonindex d984c4ffe..72352b3e8 100644--- a/vibe/i18n/en.json+++ b/vibe/i18n/en.json@@ -116,6 +116,15 @@"lifetimeTimeoutPrefix": "Watch stopped after reaching its lifetime timeout.","lifetimeTimeoutBody": "Watch '{name}' reached its lifetime timeout after {seconds} second(s).","lifetimeExpiredWithActiveFollowUp": "Watch lifetime expired while follow-up Run {run_id} was still active; the Watch retired without starting another Run.",+ "jobExitedWithCode": "The command exited with code {code}.",+ "jobTimedOut": "The command was stopped at its timeout.",+ "jobStoppedByVibeStop": "The command was stopped by `vibe stop`.",+ "jobStoppedAtLifetime": "The command was stopped when the Watch reached its lifetime.",+ "jobEndedWithoutExitCode": "The command ended without an exit code.",+ "jobRecordGone": "The command's job record {job_id} is gone, so how it ended is unknown.",+ "jobStoppedUnreadable": "The command was stopped because its job state could not be read.",+ "jobStateUnreadable": "The command's job state {job_id} could not be read, so how it ended is unknown.",+ "jobUnmanageable": "The command's job state cannot be read and the command could not be confirmed stopped; the Watch keeps it and retries.","circuitRepairPrompt": "[Avibe Harness] Watch \"{name}\" was automatically paused because its waiter exited successfully {count} times within {window} seconds. This pattern can create duplicate Agent Runs when a persistent level is reported as a new event on every cycle.\n\nWatch ID: {id}\nSuccessful exit timestamps:\n{timestamps}\nLatest exit code: 0\nLatest stdout:\n{stdout}\nLatest stderr:\n{stderr}\n\nInspect the Watch with `vibe watch show {id}` and inspect the waiter implementation. Fix it so exit 0 means one new reportable event. While the condition is unchanged or not ready, return one of this Watch's allowed retry exit codes: {retry_codes}. For a completed cycle with nothing reportable, return 64 and print `avibe-watch: no-event` to stderr. A recurring waiter must keep a durable cursor, state transition, or domain cooldown so the same level is not emitted repeatedly.\n\nAfter an unambiguous, reversible fix, verify the waiter behavior, update the Watch if needed, and resume it with `vibe watch resume {id}`. If the correct fix is uncertain or cannot be verified safely, leave the Watch paused and report the blocker. Do not process the latest stdout as an ordinary event; it was retained as diagnostic evidence for this repair.","protocolErrors": {"specTooLarge": "The watch worker specification is too large.",@@ -328,6 +337,14 @@"message": "This one-shot Task's schedule has already ended.","hint": "Set a new future time with `vibe task update {id} --at <time>`, then resume it."},+ "jobWatchResumeRefused": {+ "message": "A Watch on a running command cannot be resumed once it is paused or finished.",+ "hint": "Pausing or finishing ended its command. Run the command again to watch a new run, or remove this Watch with `vibe watch remove {id}`."+ },+ "jobWatchUpdateRefused": {+ "message": "This Watch waits on a command that is already running, so only its name, message, and lifetime can be changed.",+ "hint": "Its Session, Agent, delivery, and command were fixed when the command was handed over. Change the name, message, or lifetime with `vibe watch update {id}`."+ },"cronNumericWeekday": {"message": "Cron weekday field uses numbers: {field}","readings": "Avibe (APScheduler) reads it as {avibe}; crontab(5) reads it as {crontab}.",diff --git a/vibe/i18n/zh.json b/vibe/i18n/zh.jsonindex a40d4cc59..050bf7deb 100644--- a/vibe/i18n/zh.json+++ b/vibe/i18n/zh.json@@ -116,6 +116,15 @@"lifetimeTimeoutPrefix": "Watch 达到生命周期上限后已停止。","lifetimeTimeoutBody": "Watch「{name}」已达到 {seconds} 秒的生命周期上限。","lifetimeExpiredWithActiveFollowUp": "Watch 生命周期已到期,但后续 Run {run_id} 仍在执行;Watch 已停止,且没有再创建另一个 Run。",+ "jobExitedWithCode": "命令以退出码 {code} 结束。",+ "jobTimedOut": "命令到达超时时间,已被停止。",+ "jobStoppedByVibeStop": "命令已被 `vibe stop` 停止。",+ "jobStoppedAtLifetime": "Watch 到达生命周期上限,命令已被停止。",+ "jobEndedWithoutExitCode": "命令已结束,但没有留下退出码。",+ "jobRecordGone": "命令的作业记录 {job_id} 已不存在,无法确定它是如何结束的。",+ "jobStoppedUnreadable": "命令的作业状态无法读取,命令已被停止。",+ "jobStateUnreadable": "命令的作业状态 {job_id} 无法读取,无法确定它是如何结束的。",+ "jobUnmanageable": "命令的作业状态无法读取,也无法确认命令已停止;Watch 会继续持有它并稍后重试。","circuitRepairPrompt": "[Avibe Harness] Watch「{name}」已自动暂停:它的 waiter 在 {window} 秒内成功退出了 {count} 次。持续成立的条件如果每轮都被当作新事件,会批量制造重复 Agent Runs。\n\nWatch ID:{id}\n成功退出时间:\n{timestamps}\n最近退出码:0\n最近 stdout:\n{stdout}\n最近 stderr:\n{stderr}\n\n请先用 `vibe watch show {id}` 检查 Watch,再检查 waiter 的实现。修正后的契约应为:退出码 0 只表示一个新的、值得报告的事件;条件仍未变化或尚未就绪时,返回这个 Watch 已允许的重试退出码之一:{retry_codes}。一轮检查已经完成但没有值得报告的内容时,返回 64,并向 stderr 输出 `avibe-watch: no-event`。周期性 waiter 必须使用持久化游标、状态变化或业务冷却,避免反复报告同一个 level。\n\n只有在修复明确、可逆并且 waiter 行为已经验证后,才按需更新 Watch,并用 `vibe watch resume {id}` 恢复。如果无法安全确认修复,请保持暂停并报告阻碍。不要把最近 stdout 当作普通事件处理;它是为本次修复保留的诊断证据。","protocolErrors": {"specTooLarge": "Watch 工作进程配置过大。",@@ -328,6 +337,14 @@"message": "这个一次性 Task 的计划已经结束。","hint": "请先用 `vibe task update {id} --at <time>` 设置新的未来时间,再恢复它。"},+ "jobWatchResumeRefused": {+ "message": "等待运行中命令的 Watch 一旦暂停或结束,就无法恢复。",+ "hint": "暂停或结束时它的命令已经停止。如需继续,请重新运行该命令;也可以用 `vibe watch remove {id}` 删除这个 Watch。"+ },+ "jobWatchUpdateRefused": {+ "message": "这个 Watch 等待的是一个已在运行的命令,只能修改它的名称、消息和生命周期上限。",+ "hint": "它的 Session、Agent、投递目标和命令在命令交接时已经确定。可用 `vibe watch update {id}` 修改名称、消息或生命周期上限。"+ },"cronNumericWeekday": {"message": "cron 的星期字段使用了数字:{field}","readings": "Avibe(APScheduler)解析为 {avibe};标准 crontab(5) 解析为 {crontab}。",diff --git a/vibe/ui_server.py b/vibe/ui_server.pyindex f2bcd2eea..f1fc74770 100644--- a/vibe/ui_server.py+++ b/vibe/ui_server.py@@ -5952,6 +5952,19 @@ def _task_schedule_retired_response(exc):)+def _job_watch_resume_refused_response(exc, watch_id: str):+ from core.services import settings as settings_service++ lang = settings_service.load_config_or_default().language+ return _coded_error_response(+ exc.code,+ t("error.jobWatchResumeRefused.message", lang),+ 409,+ hint=t("error.jobWatchResumeRefused.hint", lang, id=watch_id),+ details={"watch_id": watch_id},+ )++def _show_page_error_response(exc):code = getattr(exc, "code", "invalid_show_page_request")if code == "resource_access_forbidden":@@ -12741,10 +12754,15 @@ def harness_watch_patch(watch_id: str):if "enabled" not in payload:return jsonify({"ok": False, "code": "invalid_payload", "message": "missing 'enabled'"}), 400enabled = bool(payload["enabled"])+ from storage.background import JobWatchResumeRefused+with _harness_store() as store:if not store.get_watch(watch_id):return jsonify({"ok": False, "code": "watch_not_found"}), 404- store.set_definition_enabled(watch_id, enabled, definition_type="watch")+ try:+ store.set_definition_enabled(watch_id, enabled, definition_type="watch")+ except JobWatchResumeRefused as exc:+ return _job_watch_resume_refused_response(exc, watch_id)watch = store.get_watch(watch_id)from core.inbox_events import publish_definitions_updated