"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""Executor for persisted scheduled research jobs.
The executor polls :class:`ScheduledResearchJobStore`, dispatches due jobs via
an injected async callable, and persists lifecycle/next-run updates after each
attempt. Schedule math is intentionally pure and clock-injected so tests can
exercise it without sleeping or reading wall-clock time.
Loop machinery — the tick loop, wakeup event, start/stop lifecycle and retry
backoff — is delegated to :class:`src.scheduling.kernel.SchedulerKernel`
(scheduler unification). This module keeps the scheduled-research domain: the
tick composition (stale recovery, expiry, due dispatch, delivery sweep), the
job state machine, the delivery outbox and verdict recording.
``executor_kind == "internal"`` jobs are dispatched non-blockingly through
:class:`src.scheduled_research.internal_dispatch.InternalRunSupervisor`:
registry-resolved callables run in fire-and-forget background tasks (sync
ones off the loop via ``asyncio.to_thread``) that own the terminal write, so
an hours-long internal run never freezes the serialized tick.
"""
from __future__ import annotations
import asyncio
import logging
import time
from typing import Awaitable, Callable, Optional
from src.config.accessor import get_env_config
from src.channels.bus.events import DeliveryReceipt
from src.scheduled_research.internal_dispatch import (
InternalDispatchCallback,
InternalRunSupervisor,
)
from src.scheduled_research.models import (
DeliveryRecord,
DeliveryStatus,
JobStatus,
ScheduledResearchJob,
validate_schedule,
)
from src.scheduled_research.store import ScheduledResearchJobStore
from src.scheduled_research.verdict import (
VerdictRecord,
outcome_of,
parse_verdict_section,
)
from src.scheduling.kernel import SchedulerKernel
from dataclasses import replace as _dc_replace
from src.tools.redaction import redact_internal_paths, redact_text
# --- Schedule math, moved to src.scheduling.triggers (scheduler unification) ---
# Re-exported here so existing ``from src.scheduled_research.executor import ...``
# keeps working (same convention as src/agent/loop.py's llm_usage re-export).
from src.scheduling.triggers import (
CRON_BOUNDS as CRON_BOUNDS,
CRON_SEARCH_LIMIT_DAYS as CRON_SEARCH_LIMIT_DAYS,
_CRON_SEARCH_LIMIT_DAYS as _CRON_SEARCH_LIMIT_DAYS,
_day_matches as _day_matches,
_local_wall_time_to_epoch_ms as _local_wall_time_to_epoch_ms,
_next_cron_due as _next_cron_due,
day_matches as day_matches,
local_wall_time_to_epoch_ms as local_wall_time_to_epoch_ms,
next_cron_due as next_cron_due,
next_due as next_due,
parse_cron_field as parse_cron_field,
validate_timezone as validate_timezone,
)
logger = logging.getLogger(__name__)
DEFAULT_TICK_INTERVAL_MS = 60 * 1000
SCHEDULER_ENABLED_ENV = "VIBE_TRADING_ENABLE_SCHEDULER"
_MAX_PERSISTED_ERROR_CHARS = 1000
#: A send that has not returned within this long is assumed to belong to a
#: process that is no longer running.
DEFAULT_DELIVERY_LEASE_MS = 5 * 60 * 1000
NowFn = Callable[[], int]
# A dispatcher may return the session id it enqueued into. Returning None keeps
# the pre-delivery contract working unchanged.
DispatchCallback = Callable[[ScheduledResearchJob], Awaitable[Optional[str]]]
#: session_id -> (terminal status, briefing text), or None while in flight.
BriefingReader = Callable[[str], Optional[tuple[str, str]]]
#: (channel, target, text[, delivery_format]) -> delivered.
ChannelSender = Callable[..., Awaitable[DeliveryReceipt | None]]
_TRUE_VALUES = {"1", "true", "yes", "on"}
def _now_ms() -> int:
"""Return current wall-clock time in epoch milliseconds."""
return int(time.time() * 1000)
def scheduler_enabled_from_env(value: str | None = None) -> bool:
"""Return whether the scheduled-research executor should run.
The feature is disabled by default. Pass *value* in tests to avoid mutating
process environment.
"""
if value is not None:
return value.strip().lower() in _TRUE_VALUES
return get_env_config().agent_tuning.vibe_trading_enable_scheduler
def is_due(job: ScheduledResearchJob, now_ms: int) -> bool:
"""Return whether *job* should fire at ``now_ms``.
Cancelled and failed jobs are terminal and never re-dispatched; a failed
job in particular keeps its old ``next_run_at`` (advancement may itself be
what failed), so excluding it here prevents a re-dispatch loop every tick.
Already-running jobs are left alone during live polling. Executor startup
recovers stale persisted ``RUNNING`` jobs separately.
A firing whose outbox row is still PENDING or SENDING is also left alone:
dispatch returns once the run is accepted, not once it is delivered, so a
schedule shorter than that gap would otherwise re-dispatch onto the same
row and overwrite it, orphaning the briefing a sweep still owes.
"""
if job.status in {
JobStatus.CANCELLED,
JobStatus.RUNNING,
JobStatus.FAILED,
JobStatus.EXPIRED,
}:
return False
if job.end_at is not None and now_ms > job.end_at:
return False
if job.delivery.status in {DeliveryStatus.PENDING, DeliveryStatus.SENDING}:
return False
return job.next_run_at <= now_ms
def _persisted_error(exc: Exception) -> str:
"""Return a bounded, redaction-safe error for durable job state."""
message = f"{type(exc).__name__}: {exc}"
safe = redact_text(redact_internal_paths(message)).replace("\x00", "")
if len(safe) <= _MAX_PERSISTED_ERROR_CHARS:
return safe
return f"{safe[: _MAX_PERSISTED_ERROR_CHARS - 3]}..."
class ScheduledResearchExecutor:
"""Background poller that dispatches due scheduled research jobs.
The tick loop, wakeup, start/stop lifecycle and retry backoff live in a
:class:`~src.scheduling.kernel.SchedulerKernel` instance; this class
supplies the kernel's ``on_tick`` composition (stale recovery → expiry →
due-job dispatch with per-job isolation → delivery sweep) and owns every
scheduled-research semantic (job state machine, delivery outbox, verdict
recording).
"""
def __init__(
self,
store: ScheduledResearchJobStore,
dispatch: DispatchCallback,
*,
tick_interval_ms: int = DEFAULT_TICK_INTERVAL_MS,
now_fn: NowFn = _now_ms,
enabled: bool = True,
max_consecutive_failures: int | None = None,
retry_base_delay_ms: int | None = None,
retry_max_delay_ms: int | None = None,
briefing_reader: "BriefingReader | None" = None,
channel_sender: "ChannelSender | None" = None,
delivery_lease_ms: int = DEFAULT_DELIVERY_LEASE_MS,
internal_dispatcher: "InternalDispatchCallback | None" = None,
) -> None:
"""Initialize the executor.
Args:
store: Durable scheduled job store.
dispatch: Async callable invoked once for each due job.
tick_interval_ms: Poll interval for the background loop.
now_fn: Injectable wall-clock source returning epoch milliseconds.
enabled: When false, :meth:`start` and :meth:`stop` are no-ops.
max_consecutive_failures: Dispatch failures allowed before a job
becomes terminal. Defaults to environment configuration.
retry_base_delay_ms: Base delay for exponential retry backoff.
retry_max_delay_ms: Upper bound for exponential retry backoff.
briefing_reader: Callable returning ``(terminal_status, text)`` for
a session, or ``None`` while its run is still in flight. Both
collaborators are injected rather than imported so the outbox
can be driven in a test without a session runtime or a network.
channel_sender: Async callable delivering one briefing.
delivery_lease_ms: How long a claimed row stays claimed before a
later sweep may take it over. Too short duplicates a slow
send; too long strands a briefing behind a dead process.
internal_dispatcher: Callable resolving one ``executor_kind ==
"internal"`` job to its registered zero-arg callable — in
production the exact-key allowlist lookup in
:mod:`src.scheduling.registry`. ``None`` (the default) keeps
internal jobs fail-closed: their firings take the existing
dispatch-failure path. The resolved callable never runs on
the tick loop; :class:`InternalRunSupervisor` owns the
non-blocking run and its terminal write.
Raises:
ValueError: If the retry policy is invalid.
"""
tuning = None
if None in (max_consecutive_failures, retry_base_delay_ms, retry_max_delay_ms):
tuning = get_env_config().agent_tuning
self._store = store
self._dispatch = dispatch
self._internal_dispatcher = internal_dispatcher
self._briefing_reader = briefing_reader
self._channel_sender = channel_sender
self._delivery_lease_ms = delivery_lease_ms
self._loop: asyncio.AbstractEventLoop | None = None
self._sweep_task: asyncio.Task | None = None
self._now_fn = now_fn
self._enabled = enabled
self._max_consecutive_failures = (
max_consecutive_failures
if max_consecutive_failures is not None
else tuning.vibe_trading_scheduler_max_consecutive_failures
)
self._retry_base_delay_ms = (
retry_base_delay_ms
if retry_base_delay_ms is not None
else tuning.vibe_trading_scheduler_retry_base_delay_ms
)
self._retry_max_delay_ms = (
retry_max_delay_ms
if retry_max_delay_ms is not None
else tuning.vibe_trading_scheduler_retry_max_delay_ms
)
# The kernel validates the resolved retry policy; its ValueError
# messages are byte-identical to the ones this constructor raised
# before the loop machinery moved there. The kernel never reads the
# environment — the resolved plain numbers above are the only source.
self._kernel = SchedulerKernel(
tick_interval_ms=tick_interval_ms,
now_fn=now_fn,
enabled=enabled,
max_consecutive_failures=self._max_consecutive_failures,
retry_base_delay_ms=self._retry_base_delay_ms,
retry_max_delay_ms=self._retry_max_delay_ms,
on_tick=self._on_kernel_tick,
task_name="scheduled-research-executor",
)
self._internal = InternalRunSupervisor(
store=store,
dispatcher=internal_dispatcher,
finish_run=self._finish_run,
now_fn=now_fn,
)
self._recovered_stale_running = False
@property
def is_running(self) -> bool:
"""Return whether the background loop task is active."""
return self._kernel.is_running
def start(self) -> None:
"""Start the background loop.
Idempotent. When disabled, this is a no-op.
"""
if not self._enabled or self.is_running:
return
self.recover_stale_running()
self._kernel.start()
async def stop(self) -> None:
"""Stop the background loop and wait for it to finish.
Idempotent. When disabled or not started, this is a no-op. The kernel
sets the wakeup so a sleeping loop returns at once, and cancels the
task as a fallback so shutdown never blocks for a full tick.
In-flight internal runs are then cancelled at the asyncio-task level,
never at the thread level (pinned semantics, see
:mod:`src.scheduled_research.internal_dispatch`): a sync callable
already inside ``asyncio.to_thread`` finishes harmlessly with its
result discarded, and its job stays RUNNING in the store for
``recover_stale_running`` on the next start. ``stop()`` does not wait
for those threads, so it stays bounded however long a callable blocks.
"""
await self._kernel.stop()
await self._internal.cancel_all()
async def tick(self, now_ms: int | None = None) -> None:
"""Run one poll/dispatch pass.
Args:
now_ms: Optional explicit reference time. Defaults to ``now_fn``.
"""
self.recover_stale_running()
now = self._now_fn() if now_ms is None else now_ms
self._expire_elapsed_jobs(now)
jobs = sorted(
(job for job in self._store.load().values() if is_due(job, now)),
key=lambda job: job.next_run_at,
)
for job in jobs:
# One job's unexpected persistence/lifecycle error must not starve
# every job sorted after it, tick after tick.
try:
await self._run_job(job, now)
except asyncio.CancelledError:
raise
except Exception:
logger.error("scheduled research job %s failed its run cycle", job.id, exc_info=True)
# The sweep is what makes delivery correct; the event hook only makes
# it prompt. A briefing whose hook was lost to a restart, a crash
# between "run finished" and "message sent", or a transient send error
# is picked up here on the next tick.
await self.sweep_deliveries()
def _expire_elapsed_jobs(self, now_ms: int) -> int:
"""Persist jobs whose configured end boundary has elapsed."""
jobs = self._store.load()
changed = 0
for job in jobs.values():
if job.end_at is None or now_ms <= job.end_at:
continue
if job.status in {
JobStatus.CANCELLED,
JobStatus.FAILED,
JobStatus.EXPIRED,
JobStatus.RUNNING,
}:
continue
job.status = JobStatus.EXPIRED
changed += 1
if changed:
self._store.save(jobs)
return changed
def recover_stale_running(self) -> int:
"""Reset jobs left ``RUNNING`` by a previous executor process.
This runs at most once per executor instance. Jobs that become
``RUNNING`` after this recovery step are treated as live in-flight work
and remain skipped by :func:`is_due`.
"""
if self._recovered_stale_running:
return 0
jobs = self._store.load()
recovered = 0
for job in jobs.values():
if job.status != JobStatus.RUNNING:
continue
job.status = JobStatus.PENDING
recovered += 1
logger.warning("recovering stale scheduled research job %s from running to pending", job.id)
if recovered:
self._store.save(jobs)
self._recovered_stale_running = True
return recovered
async def _on_kernel_tick(self, now_ms: int) -> None:
"""Run one executor tick as the kernel's ``on_tick`` callback.
Exceptions are isolated here (rather than left to the kernel's generic
handler) so the log message stays the executor's own; a failing tick
never kills the loop and ``asyncio.CancelledError`` propagates so
:meth:`stop` stays bounded.
"""
# Captured here rather than at construction: the executor is built
# before the server's loop exists, and request_sweep needs a loop that
# is actually running to schedule onto.
self._loop = asyncio.get_running_loop()
try:
await self.tick(now_ms)
except asyncio.CancelledError:
raise
except Exception:
logger.error("scheduled research executor tick failed", exc_info=True)
async def _run_job(self, job: ScheduledResearchJob, now_ms: int) -> None:
# The tick snapshot may be stale by the time we reach this job (an
# earlier dispatch was awaited). Re-read and confirm identity before
# marking it RUNNING so a job the user deleted or replaced in the
# meantime is not resurrected or dispatched.
current = self._store.get(job.id)
if current is None or not self._same_record(current, job) or not is_due(current, now_ms):
return
job = current
# A persisted schedule the current grammar rejects (an older release
# accepted a form since narrowed) must surface as a failed job, not as
# an exception on the way to dispatch: that would leave the record
# PENDING and due, retrying on every tick forever with nothing visible
# to the user.
try:
validate_schedule(job.schedule)
except ValueError as exc:
logger.error("scheduled research job %s has an invalid schedule", job.id, exc_info=True)
job.status = JobStatus.FAILED
job.failure_kind = "schedule"
job.last_error = _persisted_error(exc)
job.last_run_at = now_ms
self._persist_completion(job)
return
if job.executor_kind == "internal":
# Non-blocking internal dispatch: resolve → RUNNING → a
# fire-and-forget background task owns the callable and the
# terminal write. Awaiting the run here would freeze the
# serialized tick loop for hours (see internal_dispatch.py).
self._internal.launch(job, now_ms)
return
job.status = JobStatus.RUNNING
# Lifecycle writes bypass validation: this record is already persisted,
# and a write that cannot land would strand the job mid-flight.
self._store.upsert(job, validate=False)
dispatch_error: Exception | None = None
try:
session_id = await self._dispatch(job)
if isinstance(session_id, str) and session_id:
if job.delivery_channel:
# Arm the outbox in the same write that records the firing.
# The row exists before anything can be sent, so a crash can
# only ever lose the *speed* of delivery, never the fact that
# one is owed.
job.delivery = DeliveryRecord(
status=DeliveryStatus.PENDING,
session_id=session_id,
key=f"{job.id}:{session_id}:{job.delivery_channel}",
updated_at=now_ms,
)
else:
# No delivery owed, but the verdict sweep still needs the
# link from this job to its run: the session id rides the
# record with an explicit NONE so the outbox never picks it.
job.delivery = DeliveryRecord(
status=DeliveryStatus.NONE,
session_id=session_id,
updated_at=now_ms,
)
except asyncio.CancelledError:
raise
except Exception as exc:
logger.error("scheduled research dispatch failed for job %s", job.id, exc_info=True)
dispatch_error = exc
job.consecutive_failures += 1
else:
job.consecutive_failures = 0
self._finish_run(job, dispatch_error, now_ms)
def _finish_run(
self, job: ScheduledResearchJob, dispatch_error: Exception | None, now_ms: int
) -> None:
"""Record one attempt's outcome and persist the terminal state.
Shared by the prompt path — *now_ms* is the firing time, dispatch
returns at enqueue — and the internal background path, which passes
the COMPLETION time so a run longer than its own schedule interval
advances past completion instead of re-firing immediately.
"""
job.last_run_at = now_ms
try:
scheduled_next_run = next_due(job.schedule, now_ms, job.timezone)
except Exception as exc:
logger.error("scheduled research schedule advancement failed for job %s", job.id, exc_info=True)
job.status = JobStatus.FAILED
job.failure_kind = "schedule"
job.last_error = _persisted_error(exc)
self._persist_completion(job)
return
job.next_run_at = scheduled_next_run
if dispatch_error is None:
job.status = (
JobStatus.EXPIRED
if job.end_at is not None and scheduled_next_run > job.end_at
else JobStatus.COMPLETED
)
job.failure_kind = None
job.last_error = None
else:
job.failure_kind = "dispatch"
job.last_error = _persisted_error(dispatch_error)
if job.consecutive_failures >= self._max_consecutive_failures:
job.status = JobStatus.FAILED
else:
job.status = JobStatus.PENDING
retry_delay = self._retry_delay_ms(job.consecutive_failures)
job.next_run_at = max(scheduled_next_run, now_ms + retry_delay)
if job.end_at is not None and job.next_run_at > job.end_at:
job.status = JobStatus.EXPIRED
logger.warning(
"scheduled research job %s will retry after failure %d/%d at %d",
job.id,
job.consecutive_failures,
self._max_consecutive_failures,
job.next_run_at,
)
self._persist_completion(job)
def _delivery_is_eligible(self, job: ScheduledResearchJob, now_ms: int) -> bool:
"""Whether this row is a sweep's to take.
PENDING is owed. SENDING is claimed, and the claim is a lease rather
than a lock: a process that died inside the send call would otherwise
hold the row forever, so once the lease has expired the row is owed
again. The idempotency key remains the backstop for the window that
leaves.
Args:
job: The job whose outbox row is being considered.
now_ms: Reference time in epoch milliseconds.
Returns:
True when a sweep should attempt this row now.
"""
status = job.delivery.status
if status is DeliveryStatus.PENDING:
return True
if status is not DeliveryStatus.SENDING:
return False
claimed_at = job.delivery.updated_at
return claimed_at is None or now_ms - claimed_at >= self._delivery_lease_ms
def request_sweep(self) -> None:
"""Ask for a delivery sweep as soon as the loop is free.
Called from the session event listener, so it runs on whichever thread
published the event and must not block. Sweeps coalesce: one already in
flight covers anything that arrived while it was running, and the claim
makes an overlap harmless in any case. Without a running loop this is a
no-op — the periodic tick is what makes delivery correct, and this only
makes it prompt. The kernel wake shortens the loop's current sleep so
the next full tick (which ends in a sweep) follows promptly too; it
coalesces and is a no-op without a running kernel loop.
"""
self._kernel.request_wake()
loop = self._loop
if loop is None or not loop.is_running():
return
if self._sweep_task is not None and not self._sweep_task.done():
return
def _schedule() -> None:
if self._sweep_task is not None and not self._sweep_task.done():
return
self._sweep_task = loop.create_task(self._sweep_quietly())
try:
loop.call_soon_threadsafe(_schedule)
except RuntimeError:
# The loop closed between the check and the call; the next tick
# picks the row up.
pass
async def _sweep_quietly(self) -> None:
"""Run a sweep whose failure must not escape into the event path."""
try:
await self.sweep_deliveries()
except asyncio.CancelledError:
raise
except Exception:
logger.error("requested delivery sweep failed", exc_info=True)
async def sweep_deliveries(self) -> int:
"""Deliver every briefing whose run has reached a terminal state.
Idempotent and restart-safe: the outbox row is the only source of
truth, a row already ``SENT`` is never re-sent, and a row whose run is
still in flight is left for a later sweep.
Returns:
The number of rows whose state changed.
"""
if self._briefing_reader is None:
return 0
changed = 0
now = self._now_fn()
for job in list(self._store.load().values()):
if not job.delivery.session_id:
continue
if job.delivery.status is DeliveryStatus.NONE:
# No briefing is owed to a channel, but this sweep is the only
# terminal-observer a channel-less monitor has: read the run's
# end state here and persist its verdict once it lands.
changed += self._record_verdict_if_terminal(job, now)
continue
if not job.delivery_channel:
continue
if not self._delivery_is_eligible(job, now):
continue
if self._channel_sender is None:
continue
try:
outcome = self._briefing_reader(job.delivery.session_id)
except Exception as exc:
logger.error("briefing read failed for job %s", job.id, exc_info=True)
self._mark_delivery_failed(job, exc, retryable=True)
changed += 1
continue
if outcome is None:
continue # still running; a later sweep will find it
status, text = outcome
if status != "completed":
# A failed or cancelled run has no briefing to deliver. Record
# why rather than leaving the row pending forever.
self._mark_delivery_failed(
job, RuntimeError(f"run {status}"), retryable=False
)
changed += 1
continue
# Claim the row BEFORE the network call. A re-read alone would not
# help: while the first send is in flight the row still reads
# PENDING, so a concurrent sweep or the event hook would deliver
# the same briefing again.
current = self._store.get(job.id)
if current is None or current.delivery.key != job.delivery.key:
continue
if not self._delivery_is_eligible(current, now):
continue
current.delivery.status = DeliveryStatus.SENDING
current.delivery.updated_at = now
self._store.upsert(current, validate=False)
try:
if job.protect_pdf:
receipt = await self._channel_sender(
job.delivery_channel,
job.delivery_target,
text,
job.delivery_format,
True,
)
elif job.delivery_format is None:
receipt = await self._channel_sender(
job.delivery_channel, job.delivery_target, text
)
else:
receipt = await self._channel_sender(
job.delivery_channel,
job.delivery_target,
text,
job.delivery_format,
)
except asyncio.CancelledError:
raise
except Exception as exc:
logger.error("briefing delivery failed for job %s", job.id, exc_info=True)
self._mark_delivery_failed(current, exc, retryable=True)
changed += 1
continue
if receipt is None:
# Compatibility for pre-receipt injected senders used by
# embedders and older tests. Production adapters always return
# an explicit receipt through BaseChannel.send_with_receipt.
current.delivery.status = DeliveryStatus.SENT
current.delivery.provider_message_id = None
elif receipt.status == "sent":
current.delivery.status = DeliveryStatus.SENT
current.delivery.provider_message_id = receipt.provider_message_id
else:
current.delivery.status = DeliveryStatus.ACCEPTED
current.delivery.provider_message_id = receipt.provider_message_id
current.delivery.error = None
current.delivery.updated_at = self._now_fn()
# The verdict rides the same write as the terminal delivery state,
# never a second pass over the job (#1140's clobbering lesson).
if (
current.last_verdict is None
or current.last_verdict.session_id != current.delivery.session_id
):
self._record_verdict_on(current, current.delivery.session_id, text, now)
self._store.upsert(current, validate=False)
changed += 1
return changed
def _record_verdict_on(
self, job: ScheduledResearchJob, session_id: str, text: str, now_ms: int
) -> None:
"""Write the run's verdict record onto ``job``, shifting the prior one.
The previous record embeds one level only: a verdict chain any deeper
is noise for the list view.
"""
parse, items = parse_verdict_section(text)
previous = job.last_verdict
if previous is not None:
previous = _dc_replace(previous, previous=None)
job.last_verdict = VerdictRecord(
session_id=session_id,
recorded_at=now_ms,
parse=parse,
outcome=outcome_of(items),
items=items,
previous=previous,
)
def _record_verdict_if_terminal(self, job: ScheduledResearchJob, now_ms: int) -> int:
"""Persist the run's verdict once its session reaches a terminal state.
Channel-less jobs have no outbox, so this sweep is the only place their
terminal briefing is ever read. In-flight runs return ``None`` from the
briefing reader and are left for a later pass; failed or cancelled runs
produce no briefing, so the prior verdict simply stays visible as stale.
Returns:
1 when the job record changed, else 0.
"""
session_id = job.delivery.session_id
if not session_id:
return 0
if job.last_verdict is not None and job.last_verdict.session_id == session_id:
return 0 # this firing is already recorded
try:
outcome = self._briefing_reader(session_id) if self._briefing_reader else None
except Exception:
logger.error("verdict read failed for job %s", job.id, exc_info=True)
return 0
if outcome is None:
return 0
status, text = outcome
if status != "completed":
return 0
self._record_verdict_on(job, session_id, text, now_ms)
self._store.upsert(job, validate=False)
return 1
def _mark_delivery_failed(
self, job: ScheduledResearchJob, exc: Exception, *, retryable: bool
) -> None:
"""Record a delivery failure without losing which firing it belonged to.
Args:
job: The job whose outbox row failed.
exc: The failure, persisted through the same redaction the dispatch
path uses.
retryable: Whether a later sweep could still succeed. A channel
outage is; a run that finished FAILED or CANCELLED is not,
because no briefing will ever exist for it.
"""
job.delivery.error = _persisted_error(exc)
job.delivery.updated_at = self._now_fn()
if retryable:
job.delivery.attempts += 1
if job.delivery.attempts < self._max_consecutive_failures:
job.delivery.status = DeliveryStatus.PENDING
self._store.upsert(job, validate=False)
return
job.delivery.status = DeliveryStatus.FAILED
self._store.upsert(job, validate=False)
def _retry_delay_ms(self, consecutive_failures: int) -> int:
"""Return bounded exponential backoff for a dispatch failure count."""
return self._kernel.retry_delay_ms(consecutive_failures)
@staticmethod
def _same_record(current: ScheduledResearchJob, job: ScheduledResearchJob) -> bool:
"""Return whether *current* is the same scheduled run we started.
``created_at`` is assigned once at creation, so a replacement POST for
the same id (which the API stamps with a fresh ``created_at``) is
distinguishable even when the schedule is unchanged.
"""
return current.id == job.id and current.created_at == job.created_at
def _persist_completion(self, job: ScheduledResearchJob) -> None:
"""Write a finished job back, unless it was changed during dispatch.
Dispatch is awaited, so a concurrent DELETE or POST for the same id can
land while a run is in flight. Reload first: if the record is gone the
user cancelled it (do not resurrect), and if it is a different record
(replaced via POST) let the new definition own its lifecycle. Only
persist our completion when it still refers to the same scheduled run.
"""
current = self._store.get(job.id)
if current is None:
logger.info("scheduled research job %s deleted during dispatch; skipping completion write", job.id)
return
if not self._same_record(current, job):
logger.info("scheduled research job %s replaced during dispatch; skipping completion write", job.id)
return
self._store.upsert(job, validate=False)