Skip to content

"Vibe-Trading: Your Personal Trading Agent"

33.6 KB
Raw Blame
"""Executor for persisted scheduled research jobs.

The executor polls :class:`ScheduledResearchJobStore`, dispatches due jobs via
an injected async callable, and persists lifecycle/next-run updates after each
attempt. Schedule math is intentionally pure and clock-injected so tests can
exercise it without sleeping or reading wall-clock time.

Loop machinery — the tick loop, wakeup event, start/stop lifecycle and retry
backoff — is delegated to :class:`src.scheduling.kernel.SchedulerKernel`
(scheduler unification). This module keeps the scheduled-research domain: the
tick composition (stale recovery, expiry, due dispatch, delivery sweep), the
job state machine, the delivery outbox and verdict recording.

``executor_kind == "internal"`` jobs are dispatched non-blockingly through
:class:`src.scheduled_research.internal_dispatch.InternalRunSupervisor`:
registry-resolved callables run in fire-and-forget background tasks (sync
ones off the loop via ``asyncio.to_thread``) that own the terminal write, so
an hours-long internal run never freezes the serialized tick.
"""

from __future__ import annotations

import asyncio
import logging
import time
from typing import Awaitable, Callable, Optional

from src.config.accessor import get_env_config
from src.channels.bus.events import DeliveryReceipt
from src.scheduled_research.internal_dispatch import (
    InternalDispatchCallback,
    InternalRunSupervisor,
)
from src.scheduled_research.models import (
    DeliveryRecord,
    DeliveryStatus,
    JobStatus,
    ScheduledResearchJob,
    validate_schedule,
)
from src.scheduled_research.store import ScheduledResearchJobStore
from src.scheduled_research.verdict import (
    VerdictRecord,
    outcome_of,
    parse_verdict_section,
)
from src.scheduling.kernel import SchedulerKernel

from dataclasses import replace as _dc_replace
from src.tools.redaction import redact_internal_paths, redact_text

# --- Schedule math, moved to src.scheduling.triggers (scheduler unification) ---
# Re-exported here so existing ``from src.scheduled_research.executor import ...``
# keeps working (same convention as src/agent/loop.py's llm_usage re-export).
from src.scheduling.triggers import (
    CRON_BOUNDS as CRON_BOUNDS,
    CRON_SEARCH_LIMIT_DAYS as CRON_SEARCH_LIMIT_DAYS,
    _CRON_SEARCH_LIMIT_DAYS as _CRON_SEARCH_LIMIT_DAYS,
    _day_matches as _day_matches,
    _local_wall_time_to_epoch_ms as _local_wall_time_to_epoch_ms,
    _next_cron_due as _next_cron_due,
    day_matches as day_matches,
    local_wall_time_to_epoch_ms as local_wall_time_to_epoch_ms,
    next_cron_due as next_cron_due,
    next_due as next_due,
    parse_cron_field as parse_cron_field,
    validate_timezone as validate_timezone,
)

logger = logging.getLogger(__name__)

DEFAULT_TICK_INTERVAL_MS = 60 * 1000
SCHEDULER_ENABLED_ENV = "VIBE_TRADING_ENABLE_SCHEDULER"
_MAX_PERSISTED_ERROR_CHARS = 1000

#: A send that has not returned within this long is assumed to belong to a
#: process that is no longer running.
DEFAULT_DELIVERY_LEASE_MS = 5 * 60 * 1000

NowFn = Callable[[], int]
# A dispatcher may return the session id it enqueued into. Returning None keeps
# the pre-delivery contract working unchanged.
DispatchCallback = Callable[[ScheduledResearchJob], Awaitable[Optional[str]]]
#: session_id -> (terminal status, briefing text), or None while in flight.
BriefingReader = Callable[[str], Optional[tuple[str, str]]]
#: (channel, target, text[, delivery_format]) -> delivered.
ChannelSender = Callable[..., Awaitable[DeliveryReceipt | None]]

_TRUE_VALUES = {"1", "true", "yes", "on"}


def _now_ms() -> int:
    """Return current wall-clock time in epoch milliseconds."""
    return int(time.time() * 1000)


def scheduler_enabled_from_env(value: str | None = None) -> bool:
    """Return whether the scheduled-research executor should run.

    The feature is disabled by default. Pass *value* in tests to avoid mutating
    process environment.
    """
    if value is not None:
        return value.strip().lower() in _TRUE_VALUES
    return get_env_config().agent_tuning.vibe_trading_enable_scheduler


def is_due(job: ScheduledResearchJob, now_ms: int) -> bool:
    """Return whether *job* should fire at ``now_ms``.

    Cancelled and failed jobs are terminal and never re-dispatched; a failed
    job in particular keeps its old ``next_run_at`` (advancement may itself be
    what failed), so excluding it here prevents a re-dispatch loop every tick.
    Already-running jobs are left alone during live polling. Executor startup
    recovers stale persisted ``RUNNING`` jobs separately.

    A firing whose outbox row is still PENDING or SENDING is also left alone:
    dispatch returns once the run is accepted, not once it is delivered, so a
    schedule shorter than that gap would otherwise re-dispatch onto the same
    row and overwrite it, orphaning the briefing a sweep still owes.
    """
    if job.status in {
        JobStatus.CANCELLED,
        JobStatus.RUNNING,
        JobStatus.FAILED,
        JobStatus.EXPIRED,
    }:
        return False
    if job.end_at is not None and now_ms > job.end_at:
        return False
    if job.delivery.status in {DeliveryStatus.PENDING, DeliveryStatus.SENDING}:
        return False
    return job.next_run_at <= now_ms


def _persisted_error(exc: Exception) -> str:
    """Return a bounded, redaction-safe error for durable job state."""
    message = f"{type(exc).__name__}: {exc}"
    safe = redact_text(redact_internal_paths(message)).replace("\x00", "")
    if len(safe) <= _MAX_PERSISTED_ERROR_CHARS:
        return safe
    return f"{safe[: _MAX_PERSISTED_ERROR_CHARS - 3]}..."


class ScheduledResearchExecutor:
    """Background poller that dispatches due scheduled research jobs.

    The tick loop, wakeup, start/stop lifecycle and retry backoff live in a
    :class:`~src.scheduling.kernel.SchedulerKernel` instance; this class
    supplies the kernel's ``on_tick`` composition (stale recovery → expiry →
    due-job dispatch with per-job isolation → delivery sweep) and owns every
    scheduled-research semantic (job state machine, delivery outbox, verdict
    recording).
    """

    def __init__(
        self,
        store: ScheduledResearchJobStore,
        dispatch: DispatchCallback,
        *,
        tick_interval_ms: int = DEFAULT_TICK_INTERVAL_MS,
        now_fn: NowFn = _now_ms,
        enabled: bool = True,
        max_consecutive_failures: int | None = None,
        retry_base_delay_ms: int | None = None,
        retry_max_delay_ms: int | None = None,
        briefing_reader: "BriefingReader | None" = None,
        channel_sender: "ChannelSender | None" = None,
        delivery_lease_ms: int = DEFAULT_DELIVERY_LEASE_MS,
        internal_dispatcher: "InternalDispatchCallback | None" = None,
    ) -> None:
        """Initialize the executor.

        Args:
            store: Durable scheduled job store.
            dispatch: Async callable invoked once for each due job.
            tick_interval_ms: Poll interval for the background loop.
            now_fn: Injectable wall-clock source returning epoch milliseconds.
            enabled: When false, :meth:`start` and :meth:`stop` are no-ops.
            max_consecutive_failures: Dispatch failures allowed before a job
                becomes terminal. Defaults to environment configuration.
            retry_base_delay_ms: Base delay for exponential retry backoff.
            retry_max_delay_ms: Upper bound for exponential retry backoff.
            briefing_reader: Callable returning ``(terminal_status, text)`` for
                a session, or ``None`` while its run is still in flight. Both
                collaborators are injected rather than imported so the outbox
                can be driven in a test without a session runtime or a network.
            channel_sender: Async callable delivering one briefing.
            delivery_lease_ms: How long a claimed row stays claimed before a
                later sweep may take it over. Too short duplicates a slow
                send; too long strands a briefing behind a dead process.
            internal_dispatcher: Callable resolving one ``executor_kind ==
                "internal"`` job to its registered zero-arg callable — in
                production the exact-key allowlist lookup in
                :mod:`src.scheduling.registry`. ``None`` (the default) keeps
                internal jobs fail-closed: their firings take the existing
                dispatch-failure path. The resolved callable never runs on
                the tick loop; :class:`InternalRunSupervisor` owns the
                non-blocking run and its terminal write.

        Raises:
            ValueError: If the retry policy is invalid.
        """
        tuning = None
        if None in (max_consecutive_failures, retry_base_delay_ms, retry_max_delay_ms):
            tuning = get_env_config().agent_tuning
        self._store = store
        self._dispatch = dispatch
        self._internal_dispatcher = internal_dispatcher
        self._briefing_reader = briefing_reader
        self._channel_sender = channel_sender
        self._delivery_lease_ms = delivery_lease_ms
        self._loop: asyncio.AbstractEventLoop | None = None
        self._sweep_task: asyncio.Task | None = None
        self._now_fn = now_fn
        self._enabled = enabled
        self._max_consecutive_failures = (
            max_consecutive_failures
            if max_consecutive_failures is not None
            else tuning.vibe_trading_scheduler_max_consecutive_failures
        )
        self._retry_base_delay_ms = (
            retry_base_delay_ms
            if retry_base_delay_ms is not None
            else tuning.vibe_trading_scheduler_retry_base_delay_ms
        )
        self._retry_max_delay_ms = (
            retry_max_delay_ms
            if retry_max_delay_ms is not None
            else tuning.vibe_trading_scheduler_retry_max_delay_ms
        )
        # The kernel validates the resolved retry policy; its ValueError
        # messages are byte-identical to the ones this constructor raised
        # before the loop machinery moved there. The kernel never reads the
        # environment — the resolved plain numbers above are the only source.
        self._kernel = SchedulerKernel(
            tick_interval_ms=tick_interval_ms,
            now_fn=now_fn,
            enabled=enabled,
            max_consecutive_failures=self._max_consecutive_failures,
            retry_base_delay_ms=self._retry_base_delay_ms,
            retry_max_delay_ms=self._retry_max_delay_ms,
            on_tick=self._on_kernel_tick,
            task_name="scheduled-research-executor",
        )
        self._internal = InternalRunSupervisor(
            store=store,
            dispatcher=internal_dispatcher,
            finish_run=self._finish_run,
            now_fn=now_fn,
        )
        self._recovered_stale_running = False

    @property
    def is_running(self) -> bool:
        """Return whether the background loop task is active."""
        return self._kernel.is_running

    def start(self) -> None:
        """Start the background loop.

        Idempotent. When disabled, this is a no-op.
        """
        if not self._enabled or self.is_running:
            return
        self.recover_stale_running()
        self._kernel.start()

    async def stop(self) -> None:
        """Stop the background loop and wait for it to finish.

        Idempotent. When disabled or not started, this is a no-op. The kernel
        sets the wakeup so a sleeping loop returns at once, and cancels the
        task as a fallback so shutdown never blocks for a full tick.

        In-flight internal runs are then cancelled at the asyncio-task level,
        never at the thread level (pinned semantics, see
        :mod:`src.scheduled_research.internal_dispatch`): a sync callable
        already inside ``asyncio.to_thread`` finishes harmlessly with its
        result discarded, and its job stays RUNNING in the store for
        ``recover_stale_running`` on the next start. ``stop()`` does not wait
        for those threads, so it stays bounded however long a callable blocks.
        """
        await self._kernel.stop()
        await self._internal.cancel_all()

    async def tick(self, now_ms: int | None = None) -> None:
        """Run one poll/dispatch pass.

        Args:
            now_ms: Optional explicit reference time. Defaults to ``now_fn``.
        """
        self.recover_stale_running()
        now = self._now_fn() if now_ms is None else now_ms
        self._expire_elapsed_jobs(now)
        jobs = sorted(
            (job for job in self._store.load().values() if is_due(job, now)),
            key=lambda job: job.next_run_at,
        )
        for job in jobs:
            # One job's unexpected persistence/lifecycle error must not starve
            # every job sorted after it, tick after tick.
            try:
                await self._run_job(job, now)
            except asyncio.CancelledError:
                raise
            except Exception:
                logger.error("scheduled research job %s failed its run cycle", job.id, exc_info=True)

        # The sweep is what makes delivery correct; the event hook only makes
        # it prompt. A briefing whose hook was lost to a restart, a crash
        # between "run finished" and "message sent", or a transient send error
        # is picked up here on the next tick.
        await self.sweep_deliveries()

    def _expire_elapsed_jobs(self, now_ms: int) -> int:
        """Persist jobs whose configured end boundary has elapsed."""
        jobs = self._store.load()
        changed = 0
        for job in jobs.values():
            if job.end_at is None or now_ms <= job.end_at:
                continue
            if job.status in {
                JobStatus.CANCELLED,
                JobStatus.FAILED,
                JobStatus.EXPIRED,
                JobStatus.RUNNING,
            }:
                continue
            job.status = JobStatus.EXPIRED
            changed += 1
        if changed:
            self._store.save(jobs)
        return changed

    def recover_stale_running(self) -> int:
        """Reset jobs left ``RUNNING`` by a previous executor process.

        This runs at most once per executor instance. Jobs that become
        ``RUNNING`` after this recovery step are treated as live in-flight work
        and remain skipped by :func:`is_due`.
        """
        if self._recovered_stale_running:
            return 0

        jobs = self._store.load()
        recovered = 0
        for job in jobs.values():
            if job.status != JobStatus.RUNNING:
                continue
            job.status = JobStatus.PENDING
            recovered += 1
            logger.warning("recovering stale scheduled research job %s from running to pending", job.id)

        if recovered:
            self._store.save(jobs)
        self._recovered_stale_running = True
        return recovered

    async def _on_kernel_tick(self, now_ms: int) -> None:
        """Run one executor tick as the kernel's ``on_tick`` callback.

        Exceptions are isolated here (rather than left to the kernel's generic
        handler) so the log message stays the executor's own; a failing tick
        never kills the loop and ``asyncio.CancelledError`` propagates so
        :meth:`stop` stays bounded.
        """
        # Captured here rather than at construction: the executor is built
        # before the server's loop exists, and request_sweep needs a loop that
        # is actually running to schedule onto.
        self._loop = asyncio.get_running_loop()
        try:
            await self.tick(now_ms)
        except asyncio.CancelledError:
            raise
        except Exception:
            logger.error("scheduled research executor tick failed", exc_info=True)

    async def _run_job(self, job: ScheduledResearchJob, now_ms: int) -> None:
        # The tick snapshot may be stale by the time we reach this job (an
        # earlier dispatch was awaited). Re-read and confirm identity before
        # marking it RUNNING so a job the user deleted or replaced in the
        # meantime is not resurrected or dispatched.
        current = self._store.get(job.id)
        if current is None or not self._same_record(current, job) or not is_due(current, now_ms):
            return
        job = current

        # A persisted schedule the current grammar rejects (an older release
        # accepted a form since narrowed) must surface as a failed job, not as
        # an exception on the way to dispatch: that would leave the record
        # PENDING and due, retrying on every tick forever with nothing visible
        # to the user.
        try:
            validate_schedule(job.schedule)
        except ValueError as exc:
            logger.error("scheduled research job %s has an invalid schedule", job.id, exc_info=True)
            job.status = JobStatus.FAILED
            job.failure_kind = "schedule"
            job.last_error = _persisted_error(exc)
            job.last_run_at = now_ms
            self._persist_completion(job)
            return

        if job.executor_kind == "internal":
            # Non-blocking internal dispatch: resolve → RUNNING → a
            # fire-and-forget background task owns the callable and the
            # terminal write. Awaiting the run here would freeze the
            # serialized tick loop for hours (see internal_dispatch.py).
            self._internal.launch(job, now_ms)
            return

        job.status = JobStatus.RUNNING
        # Lifecycle writes bypass validation: this record is already persisted,
        # and a write that cannot land would strand the job mid-flight.
        self._store.upsert(job, validate=False)

        dispatch_error: Exception | None = None
        try:
            session_id = await self._dispatch(job)
            if isinstance(session_id, str) and session_id:
                if job.delivery_channel:
                    # Arm the outbox in the same write that records the firing.
                    # The row exists before anything can be sent, so a crash can
                    # only ever lose the *speed* of delivery, never the fact that
                    # one is owed.
                    job.delivery = DeliveryRecord(
                        status=DeliveryStatus.PENDING,
                        session_id=session_id,
                        key=f"{job.id}:{session_id}:{job.delivery_channel}",
                        updated_at=now_ms,
                    )
                else:
                    # No delivery owed, but the verdict sweep still needs the
                    # link from this job to its run: the session id rides the
                    # record with an explicit NONE so the outbox never picks it.
                    job.delivery = DeliveryRecord(
                        status=DeliveryStatus.NONE,
                        session_id=session_id,
                        updated_at=now_ms,
                    )
        except asyncio.CancelledError:
            raise
        except Exception as exc:
            logger.error("scheduled research dispatch failed for job %s", job.id, exc_info=True)
            dispatch_error = exc
            job.consecutive_failures += 1
        else:
            job.consecutive_failures = 0

        self._finish_run(job, dispatch_error, now_ms)

    def _finish_run(
        self, job: ScheduledResearchJob, dispatch_error: Exception | None, now_ms: int
    ) -> None:
        """Record one attempt's outcome and persist the terminal state.

        Shared by the prompt path — *now_ms* is the firing time, dispatch
        returns at enqueue — and the internal background path, which passes
        the COMPLETION time so a run longer than its own schedule interval
        advances past completion instead of re-firing immediately.
        """
        job.last_run_at = now_ms
        try:
            scheduled_next_run = next_due(job.schedule, now_ms, job.timezone)
        except Exception as exc:
            logger.error("scheduled research schedule advancement failed for job %s", job.id, exc_info=True)
            job.status = JobStatus.FAILED
            job.failure_kind = "schedule"
            job.last_error = _persisted_error(exc)
            self._persist_completion(job)
            return

        job.next_run_at = scheduled_next_run
        if dispatch_error is None:
            job.status = (
                JobStatus.EXPIRED
                if job.end_at is not None and scheduled_next_run > job.end_at
                else JobStatus.COMPLETED
            )
            job.failure_kind = None
            job.last_error = None
        else:
            job.failure_kind = "dispatch"
            job.last_error = _persisted_error(dispatch_error)
            if job.consecutive_failures >= self._max_consecutive_failures:
                job.status = JobStatus.FAILED
            else:
                job.status = JobStatus.PENDING
                retry_delay = self._retry_delay_ms(job.consecutive_failures)
                job.next_run_at = max(scheduled_next_run, now_ms + retry_delay)
                if job.end_at is not None and job.next_run_at > job.end_at:
                    job.status = JobStatus.EXPIRED
                logger.warning(
                    "scheduled research job %s will retry after failure %d/%d at %d",
                    job.id,
                    job.consecutive_failures,
                    self._max_consecutive_failures,
                    job.next_run_at,
                )
        self._persist_completion(job)

    def _delivery_is_eligible(self, job: ScheduledResearchJob, now_ms: int) -> bool:
        """Whether this row is a sweep's to take.

        PENDING is owed. SENDING is claimed, and the claim is a lease rather
        than a lock: a process that died inside the send call would otherwise
        hold the row forever, so once the lease has expired the row is owed
        again. The idempotency key remains the backstop for the window that
        leaves.

        Args:
            job: The job whose outbox row is being considered.
            now_ms: Reference time in epoch milliseconds.

        Returns:
            True when a sweep should attempt this row now.
        """
        status = job.delivery.status
        if status is DeliveryStatus.PENDING:
            return True
        if status is not DeliveryStatus.SENDING:
            return False
        claimed_at = job.delivery.updated_at
        return claimed_at is None or now_ms - claimed_at >= self._delivery_lease_ms

    def request_sweep(self) -> None:
        """Ask for a delivery sweep as soon as the loop is free.

        Called from the session event listener, so it runs on whichever thread
        published the event and must not block. Sweeps coalesce: one already in
        flight covers anything that arrived while it was running, and the claim
        makes an overlap harmless in any case. Without a running loop this is a
        no-op — the periodic tick is what makes delivery correct, and this only
        makes it prompt. The kernel wake shortens the loop's current sleep so
        the next full tick (which ends in a sweep) follows promptly too; it
        coalesces and is a no-op without a running kernel loop.
        """
        self._kernel.request_wake()
        loop = self._loop
        if loop is None or not loop.is_running():
            return
        if self._sweep_task is not None and not self._sweep_task.done():
            return

        def _schedule() -> None:
            if self._sweep_task is not None and not self._sweep_task.done():
                return
            self._sweep_task = loop.create_task(self._sweep_quietly())

        try:
            loop.call_soon_threadsafe(_schedule)
        except RuntimeError:
            # The loop closed between the check and the call; the next tick
            # picks the row up.
            pass

    async def _sweep_quietly(self) -> None:
        """Run a sweep whose failure must not escape into the event path."""
        try:
            await self.sweep_deliveries()
        except asyncio.CancelledError:
            raise
        except Exception:
            logger.error("requested delivery sweep failed", exc_info=True)

    async def sweep_deliveries(self) -> int:
        """Deliver every briefing whose run has reached a terminal state.

        Idempotent and restart-safe: the outbox row is the only source of
        truth, a row already ``SENT`` is never re-sent, and a row whose run is
        still in flight is left for a later sweep.

        Returns:
            The number of rows whose state changed.
        """
        if self._briefing_reader is None:
            return 0

        changed = 0
        now = self._now_fn()
        for job in list(self._store.load().values()):
            if not job.delivery.session_id:
                continue
            if job.delivery.status is DeliveryStatus.NONE:
                # No briefing is owed to a channel, but this sweep is the only
                # terminal-observer a channel-less monitor has: read the run's
                # end state here and persist its verdict once it lands.
                changed += self._record_verdict_if_terminal(job, now)
                continue
            if not job.delivery_channel:
                continue
            if not self._delivery_is_eligible(job, now):
                continue
            if self._channel_sender is None:
                continue
            try:
                outcome = self._briefing_reader(job.delivery.session_id)
            except Exception as exc:
                logger.error("briefing read failed for job %s", job.id, exc_info=True)
                self._mark_delivery_failed(job, exc, retryable=True)
                changed += 1
                continue
            if outcome is None:
                continue  # still running; a later sweep will find it

            status, text = outcome
            if status != "completed":
                # A failed or cancelled run has no briefing to deliver. Record
                # why rather than leaving the row pending forever.
                self._mark_delivery_failed(
                    job, RuntimeError(f"run {status}"), retryable=False
                )
                changed += 1
                continue

            # Claim the row BEFORE the network call. A re-read alone would not
            # help: while the first send is in flight the row still reads
            # PENDING, so a concurrent sweep or the event hook would deliver
            # the same briefing again.
            current = self._store.get(job.id)
            if current is None or current.delivery.key != job.delivery.key:
                continue
            if not self._delivery_is_eligible(current, now):
                continue
            current.delivery.status = DeliveryStatus.SENDING
            current.delivery.updated_at = now
            self._store.upsert(current, validate=False)

            try:
                if job.protect_pdf:
                    receipt = await self._channel_sender(
                        job.delivery_channel,
                        job.delivery_target,
                        text,
                        job.delivery_format,
                        True,
                    )
                elif job.delivery_format is None:
                    receipt = await self._channel_sender(
                        job.delivery_channel, job.delivery_target, text
                    )
                else:
                    receipt = await self._channel_sender(
                        job.delivery_channel,
                        job.delivery_target,
                        text,
                        job.delivery_format,
                    )
            except asyncio.CancelledError:
                raise
            except Exception as exc:
                logger.error("briefing delivery failed for job %s", job.id, exc_info=True)
                self._mark_delivery_failed(current, exc, retryable=True)
                changed += 1
                continue

            if receipt is None:
                # Compatibility for pre-receipt injected senders used by
                # embedders and older tests. Production adapters always return
                # an explicit receipt through BaseChannel.send_with_receipt.
                current.delivery.status = DeliveryStatus.SENT
                current.delivery.provider_message_id = None
            elif receipt.status == "sent":
                current.delivery.status = DeliveryStatus.SENT
                current.delivery.provider_message_id = receipt.provider_message_id
            else:
                current.delivery.status = DeliveryStatus.ACCEPTED
                current.delivery.provider_message_id = receipt.provider_message_id
            current.delivery.error = None
            current.delivery.updated_at = self._now_fn()
            # The verdict rides the same write as the terminal delivery state,
            # never a second pass over the job (#1140's clobbering lesson).
            if (
                current.last_verdict is None
                or current.last_verdict.session_id != current.delivery.session_id
            ):
                self._record_verdict_on(current, current.delivery.session_id, text, now)
            self._store.upsert(current, validate=False)
            changed += 1
        return changed

    def _record_verdict_on(
        self, job: ScheduledResearchJob, session_id: str, text: str, now_ms: int
    ) -> None:
        """Write the run's verdict record onto ``job``, shifting the prior one.

        The previous record embeds one level only: a verdict chain any deeper
        is noise for the list view.
        """
        parse, items = parse_verdict_section(text)
        previous = job.last_verdict
        if previous is not None:
            previous = _dc_replace(previous, previous=None)
        job.last_verdict = VerdictRecord(
            session_id=session_id,
            recorded_at=now_ms,
            parse=parse,
            outcome=outcome_of(items),
            items=items,
            previous=previous,
        )

    def _record_verdict_if_terminal(self, job: ScheduledResearchJob, now_ms: int) -> int:
        """Persist the run's verdict once its session reaches a terminal state.

        Channel-less jobs have no outbox, so this sweep is the only place their
        terminal briefing is ever read. In-flight runs return ``None`` from the
        briefing reader and are left for a later pass; failed or cancelled runs
        produce no briefing, so the prior verdict simply stays visible as stale.

        Returns:
            1 when the job record changed, else 0.
        """
        session_id = job.delivery.session_id
        if not session_id:
            return 0
        if job.last_verdict is not None and job.last_verdict.session_id == session_id:
            return 0  # this firing is already recorded
        try:
            outcome = self._briefing_reader(session_id) if self._briefing_reader else None
        except Exception:
            logger.error("verdict read failed for job %s", job.id, exc_info=True)
            return 0
        if outcome is None:
            return 0
        status, text = outcome
        if status != "completed":
            return 0
        self._record_verdict_on(job, session_id, text, now_ms)
        self._store.upsert(job, validate=False)
        return 1

    def _mark_delivery_failed(
        self, job: ScheduledResearchJob, exc: Exception, *, retryable: bool
    ) -> None:
        """Record a delivery failure without losing which firing it belonged to.

        Args:
            job: The job whose outbox row failed.
            exc: The failure, persisted through the same redaction the dispatch
                path uses.
            retryable: Whether a later sweep could still succeed. A channel
                outage is; a run that finished FAILED or CANCELLED is not,
                because no briefing will ever exist for it.
        """
        job.delivery.error = _persisted_error(exc)
        job.delivery.updated_at = self._now_fn()
        if retryable:
            job.delivery.attempts += 1
            if job.delivery.attempts < self._max_consecutive_failures:
                job.delivery.status = DeliveryStatus.PENDING
                self._store.upsert(job, validate=False)
                return
        job.delivery.status = DeliveryStatus.FAILED
        self._store.upsert(job, validate=False)

    def _retry_delay_ms(self, consecutive_failures: int) -> int:
        """Return bounded exponential backoff for a dispatch failure count."""
        return self._kernel.retry_delay_ms(consecutive_failures)

    @staticmethod
    def _same_record(current: ScheduledResearchJob, job: ScheduledResearchJob) -> bool:
        """Return whether *current* is the same scheduled run we started.

        ``created_at`` is assigned once at creation, so a replacement POST for
        the same id (which the API stamps with a fresh ``created_at``) is
        distinguishable even when the schedule is unchanged.
        """
        return current.id == job.id and current.created_at == job.created_at

    def _persist_completion(self, job: ScheduledResearchJob) -> None:
        """Write a finished job back, unless it was changed during dispatch.

        Dispatch is awaited, so a concurrent DELETE or POST for the same id can
        land while a run is in flight. Reload first: if the record is gone the
        user cancelled it (do not resurrect), and if it is a different record
        (replaced via POST) let the new definition own its lifecycle. Only
        persist our completion when it still refers to the same scheduled run.
        """
        current = self._store.get(job.id)
        if current is None:
            logger.info("scheduled research job %s deleted during dispatch; skipping completion write", job.id)
            return
        if not self._same_record(current, job):
            logger.info("scheduled research job %s replaced during dispatch; skipping completion write", job.id)
            return
        self._store.upsert(job, validate=False)