Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

37.9 KB
Raw Blame
"""Controller-owned scheduling for passive durable-work recovery lanes."""

from __future__ import annotations

import asyncio
import logging
import threading
import time
from collections.abc import Awaitable, Callable
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from enum import Enum
from functools import partial
from typing import Any, NoReturn, Protocol, TypeVar

from core.inbox_events import (
    DEFINITIONS_UPDATED_EVENT,
    QUEUE_UPDATED_EVENT,
    RUNS_UPDATED_EVENT,
    VAULTS_UPDATED_EVENT,
    bus,
)
from vibe import runtime

logger = logging.getLogger(__name__)

_T = TypeVar("_T")


async def land_step(step: Awaitable[_T]) -> tuple[asyncio.Future[_T], bool]:
    """Wait for one store step to finish, whatever cancels the caller meanwhile.

    A Python thread cannot be cancelled safely, and a step a cancellation split would
    leave what it committed unowned, so the step is never cancelled: every
    cancellation that reaches the caller meanwhile is caught instead. Returns the
    finished step, whose ``result()`` gives its outcome, and whether a cancellation
    was caught, for the caller to answer from that outcome.
    """

    future = asyncio.ensure_future(step)
    cancelled = False
    while not future.done():
        try:
            # ``wait`` never cancels the step, and raises only the caller's cancellation.
            await asyncio.wait((future,))
        except asyncio.CancelledError:
            cancelled = True
    return future, cancelled


def step_failed(step: asyncio.Future[Any]) -> bool:
    return step.cancelled() or step.exception() is not None


def answer_caught_cancellation(step: asyncio.Future[Any]) -> NoReturn:
    """Answer a cancellation ``land_step`` caught, now.

    A failed step has no result to hand back, so this is also how a caller that keeps
    a landed step's result answers a failed one: its error is logged and kept as the
    cancellation's cause, never left to replace the cancellation.
    """

    error = None if step.cancelled() else step.exception()
    if error is not None:
        logger.warning(
            "A store step failed while its caller was being cancelled; answering the cancellation",
            exc_info=error,
        )
    raise asyncio.CancelledError from error


def rearm_caught_cancellation() -> None:
    """Deliver a cancellation ``land_step`` caught, but the caller did not answer, at
    the caller's next await -- where it would have landed had the step run on the loop.

    ``uncancel`` first (Python 3.11+, absent on 3.10, which counts no requests), so the
    request is counted once, as an ``asyncio.timeout`` around the caller expects.
    """

    current = asyncio.current_task()
    if current is None:
        return
    uncancel = getattr(current, "uncancel", None)
    if uncancel is not None:
        uncancel()
    current.cancel()


_DEFAULT_RECONCILE_INTERVAL_SECONDS = 30.0
_DEFAULT_LANE_CAPACITY = 4
_DEFAULT_SCAN_PAGE_SIZE = 32
_DEFAULT_SCAN_CONTINUATION_PAGES = 8
_DEFAULT_RETRY_BACKOFF_SECONDS = 1.0
_DEFAULT_OVERDUE_SECONDS = 30.0


class RuntimeWorkLane(str, Enum):
    SESSION_DELIVERIES = "delivery_queue_recovery"
    TASK_DEFINITIONS = "task_definitions"
    WATCH_DEFINITIONS = "watch_definitions"
    REQUESTS = "requests"
    RUN_CALLBACKS = "run_callbacks"
    VAULT_CALLBACKS = "vault_callbacks"
    ACTIVITY_OUTPUTS = "activity_outputs"
    FAILURE_NOTICES = "failure_notices"
    STALE_RUNS = "stale_runs"


@dataclass(frozen=True)
class RuntimeWorkItem:
    partition_key: str
    observation: Any
    cursor_key: str | None = None
    rearm_after_process: bool = True
    defer_until_partition_release: bool = False
    cursor_only: bool = False


class RuntimeWorkHandler(Protocol):
    """A lane re-reader and its existing guarded owner entry point.

    ``scan`` returns a bounded page ordered by stable partition key, strictly
    after the exclusive cursor when one is supplied.
    """

    def scan(
        self,
        *,
        limit: int,
        occupied: frozenset[str],
        cursor: str | None,
    ) -> tuple[list[RuntimeWorkItem], bool]: ...

    async def process(self, item: RuntimeWorkItem) -> bool | None: ...


@dataclass(frozen=True)
class RuntimeWorkRegistrationToken:
    lane: RuntimeWorkLane
    generation: int


@dataclass
class _Registration:
    token: RuntimeWorkRegistrationToken
    handler: RuntimeWorkHandler
    event: asyncio.Event = field(default_factory=asyncio.Event)
    coordinator: asyncio.Task[None] | None = None
    workers: dict[str, asyncio.Task[Any]] = field(default_factory=dict)
    worker_started_at: dict[str, float] = field(default_factory=dict)
    backoff_deadlines: dict[str, float] = field(default_factory=dict)
    backoff_task: asyncio.Task[None] | None = None
    expired_backoff_rewind_pending: bool = False
    scan_continuation_task: asyncio.Task[None] | None = None
    rewind_after_partitions: set[str] = field(default_factory=set)
    scan_started_at: float | None = None
    scan_cursor: str | None = None
    rewind_requested: bool = False
    unregister_task: asyncio.Task[None] | None = None
    live: bool = True


_EVENT_LANES = {
    QUEUE_UPDATED_EVENT: (RuntimeWorkLane.SESSION_DELIVERIES,),
    RUNS_UPDATED_EVENT: (
        RuntimeWorkLane.REQUESTS,
        RuntimeWorkLane.RUN_CALLBACKS,
        RuntimeWorkLane.FAILURE_NOTICES,
        RuntimeWorkLane.STALE_RUNS,
    ),
    VAULTS_UPDATED_EVENT: (RuntimeWorkLane.VAULT_CALLBACKS,),
    DEFINITIONS_UPDATED_EVENT: (
        RuntimeWorkLane.TASK_DEFINITIONS,
        RuntimeWorkLane.WATCH_DEFINITIONS,
    ),
}


class RuntimeWorkSupervisor:
    """Coalesce hints while durable stores retain all item authority."""

    def __init__(
        self,
        *,
        reconcile_interval: float = _DEFAULT_RECONCILE_INTERVAL_SECONDS,
        lane_capacity: int = _DEFAULT_LANE_CAPACITY,
        scan_page_size: int = _DEFAULT_SCAN_PAGE_SIZE,
        scan_continuation_pages: int = _DEFAULT_SCAN_CONTINUATION_PAGES,
        retry_backoff: float = _DEFAULT_RETRY_BACKOFF_SECONDS,
        retry_wait: Callable[[float], Awaitable[None]] | None = None,
        reconcile_wait: Callable[[float], Awaitable[None]] | None = None,
        overdue_seconds: float = _DEFAULT_OVERDUE_SECONDS,
        monotonic: Callable[[], float] = time.monotonic,
        on_lease_lost: Callable[[], None] | None = None,
    ) -> None:
        self._reconcile_interval = max(0.001, float(reconcile_interval))
        self._lane_capacity = max(1, int(lane_capacity))
        self._scan_page_size = max(1, int(scan_page_size))
        self._scan_continuation_pages = max(1, int(scan_continuation_pages))
        self._backoff_partition_limit = (
            min(self._lane_capacity, self._scan_page_size)
            * self._scan_continuation_pages
        )
        self._retry_backoff = max(0.001, float(retry_backoff))
        self._retry_wait = retry_wait or asyncio.sleep
        self._reconcile_wait = reconcile_wait or asyncio.sleep
        self._overdue_seconds = max(0.001, float(overdue_seconds))
        self._monotonic = monotonic
        self._on_lease_lost = on_lease_lost
        self._registrations: dict[RuntimeWorkLane, _Registration] = {}
        self._generations: dict[RuntimeWorkLane, int] = {}
        self._loop: asyncio.AbstractEventLoop | None = None
        self._active = False
        self._quiescing = False
        self._stopping = False
        self._subscription_id: int | None = None
        self._reconcile_task: asyncio.Task[None] | None = None
        self._stop_task: asyncio.Task[None] | None = None
        self._delayed_wakes: dict[
            RuntimeWorkRegistrationToken,
            asyncio.TimerHandle,
        ] = {}
        self._pending_lock = threading.Lock()
        self._pending_lanes: set[RuntimeWorkLane] = set()
        self._pending_rewinds: set[RuntimeWorkLane] = set()
        self._requires_service_lease = runtime.service_instance_lock_attached_to_process()
        self._sync_executor = ThreadPoolExecutor(
            max_workers=max(8, self._lane_capacity * len(RuntimeWorkLane)),
            thread_name_prefix="runtime-work",
        )
        self._sync_futures: set[asyncio.Future[Any]] = set()
        self._sync_executor_stopped = False

    def register(
        self,
        lane: RuntimeWorkLane,
        handler: RuntimeWorkHandler,
    ) -> RuntimeWorkRegistrationToken:
        if self._quiescing or self._stopping:
            raise RuntimeError("runtime work supervisor is stopping")
        existing = self._registrations.get(lane)
        if existing is not None:
            raise RuntimeError(f"runtime work lane already registered: {lane.value}")
        generation = self._generations.get(lane, 0) + 1
        self._generations[lane] = generation
        token = RuntimeWorkRegistrationToken(lane=lane, generation=generation)
        registration = _Registration(token=token, handler=handler)
        self._registrations[lane] = registration
        if self._active:
            self._start_registration(registration)
            registration.event.set()
        return token

    async def unregister(self, token: RuntimeWorkRegistrationToken) -> None:
        await self.begin_unregister(token)

    def begin_unregister(
        self,
        token: RuntimeWorkRegistrationToken,
    ) -> asyncio.Task[None]:
        """Invalidate a registration synchronously, then join it asynchronously."""

        registration = self._registrations.get(token.lane)
        if registration is None or registration.token != token:
            return asyncio.create_task(asyncio.sleep(0))
        if registration.unregister_task is not None:
            return registration.unregister_task
        # Invalidate the generation before joining it. Worker completion can no
        # longer re-arm this lane or a later replacement registration.
        registration.live = False
        registration.event.set()
        delayed = self._delayed_wakes.pop(token, None)
        if delayed is not None:
            delayed.cancel()
        registration.unregister_task = asyncio.create_task(
            self._finish_unregister(registration),
            name=f"runtime-work-unregister:{token.lane.value}",
        )
        return registration.unregister_task

    async def _finish_unregister(self, registration: _Registration) -> None:
        coordinator = registration.coordinator
        if coordinator is not None:
            await asyncio.gather(coordinator, return_exceptions=True)
        workers = tuple(registration.workers.values())
        if workers:
            await asyncio.gather(*workers, return_exceptions=True)
        backoff = registration.backoff_task
        if backoff is not None:
            backoff.cancel()
            await asyncio.gather(backoff, return_exceptions=True)
        registration.backoff_deadlines.clear()
        continuation = registration.scan_continuation_task
        if continuation is not None:
            continuation.cancel()
            await asyncio.gather(continuation, return_exceptions=True)
        if self._registrations.get(registration.token.lane) is registration:
            self._registrations.pop(registration.token.lane, None)

    async def activate(self) -> None:
        if self._active:
            return
        if self._stopping:
            raise RuntimeError("runtime work supervisor is stopping")
        if self._quiescing:
            raise RuntimeError("runtime work supervisor is quiescing")
        self._loop = asyncio.get_running_loop()
        self._active = True
        self._subscription_id = bus.subscribe_callback(self._on_bus_event)
        for registration in tuple(self._registrations.values()):
            self._start_registration(registration)
            registration.event.set()
        with self._pending_lock:
            pending = tuple(self._pending_lanes)
            rewinds = tuple(self._pending_rewinds)
            self._pending_lanes.clear()
            self._pending_rewinds.clear()
        self._notify_on_loop(pending)
        self._notify_on_loop(rewinds, True)
        self._reconcile_task = asyncio.create_task(
            self._reconcile_loop(),
            name="runtime-work-reconcile",
        )

    def notify(
        self,
        *lanes: RuntimeWorkLane,
        reset_cursor: bool = False,
    ) -> None:
        if not lanes:
            return
        if self._quiescing or self._stopping:
            return
        loop = self._loop
        if not self._active or loop is None or loop.is_closed():
            with self._pending_lock:
                self._pending_lanes.update(lanes)
                if reset_cursor:
                    self._pending_rewinds.update(lanes)
            return
        try:
            current = asyncio.get_running_loop()
        except RuntimeError:
            current = None
        if current is loop:
            self._notify_on_loop(lanes, reset_cursor)
            return
        try:
            loop.call_soon_threadsafe(
                self._notify_on_loop,
                tuple(lanes),
                reset_cursor,
            )
        except RuntimeError:
            # A closing generation may lose a hint; the next controller's
            # startup reconciliation is the durable recovery boundary.
            pass

    def notify_after(
        self,
        token: RuntimeWorkRegistrationToken,
        delay: float,
    ) -> None:
        """Coalesce a generation-scoped future eligibility wake."""

        loop = self._loop
        if not self._active or loop is None or loop.is_closed():
            return
        bounded_delay = max(0.0, float(delay))
        try:
            current = asyncio.get_running_loop()
        except RuntimeError:
            current = None
        if current is loop:
            self._notify_after_on_loop(token, bounded_delay)
            return
        try:
            loop.call_soon_threadsafe(
                self._notify_after_on_loop,
                token,
                bounded_delay,
            )
        except RuntimeError:
            pass

    def _notify_after_on_loop(
        self,
        token: RuntimeWorkRegistrationToken,
        delay: float,
    ) -> None:
        registration = self._registrations.get(token.lane)
        if (
            registration is None
            or registration.token != token
            or not registration.live
            or not self._active
            or self._stopping
        ):
            return
        if delay <= 0:
            registration.event.set()
            return
        loop = self._loop
        if loop is None:
            return
        deadline = loop.time() + delay
        existing = self._delayed_wakes.get(token)
        if existing is not None and not existing.cancelled():
            if existing.when() <= deadline:
                return
            existing.cancel()
        self._delayed_wakes[token] = loop.call_at(
            deadline,
            self._fire_delayed_wake,
            token,
        )

    def _fire_delayed_wake(self, token: RuntimeWorkRegistrationToken) -> None:
        self._delayed_wakes.pop(token, None)
        registration = self._registrations.get(token.lane)
        if (
            registration is not None
            and registration.token == token
            and registration.live
            and self._active
            and not self._stopping
        ):
            registration.rewind_requested = True
            registration.event.set()

    async def run_in_partition(
        self,
        lane: RuntimeWorkLane,
        partition_key: str,
        operation: Callable[[], Awaitable[_T]],
    ) -> _T:
        """Run a live domain owner under the same lease as recovered work.

        Activity output has both a live consumer and a restart consumer.  The
        durable Registry remains authoritative; this lease only prevents those
        two consumers from claiming or emitting for one runtime concurrently.
        """

        partition = str(partition_key or "").strip()
        registration = self._registrations.get(lane)
        loop = self._loop
        if not partition:
            return await operation()
        if self._quiescing or self._stopping:
            raise RuntimeError("runtime work partition is stopping")
        if registration is None:
            if self._active:
                raise RuntimeError("runtime work partition is unavailable")
            return await operation()
        if not registration.live:
            raise RuntimeError("runtime work partition is stopping")
        if not self._active or loop is None:
            return await operation()
        if asyncio.get_running_loop() is not loop:
            raise RuntimeError("runtime work partitions belong to the controller loop")

        current = asyncio.current_task()
        assert current is not None
        owner_task: asyncio.Task[_T] | None = None
        while True:
            self._require_live_partition_admission(registration)
            owner = registration.workers.get(partition)
            if owner is current:
                return await operation()
            if owner is None:
                owner_task = asyncio.create_task(
                    operation(),
                    name=f"runtime-work-live:{lane.value}:{partition}",
                )
                registration.workers[partition] = owner_task
                registration.worker_started_at[partition] = self._monotonic()
                owner_task.add_done_callback(
                    partial(
                        self._release_live_partition,
                        registration,
                        partition,
                    )
                )
                break
            await asyncio.gather(asyncio.shield(owner), return_exceptions=True)
            if owner.done() and registration.workers.get(partition) is owner:
                registration.workers.pop(partition, None)
                registration.worker_started_at.pop(partition, None)
                self._rewind_after_partition_release(registration, partition)

        assert owner_task is not None
        return await asyncio.shield(owner_task)

    def _require_live_partition_admission(
        self,
        registration: _Registration,
    ) -> None:
        if (
            not registration.live
            or not self._active
            or self._quiescing
            or self._stopping
        ):
            raise RuntimeError("runtime work partition is stopping")
        if not self._owns_service_instance():
            self._stop_for_lost_lease()
            raise RuntimeError("runtime work partition is stopping")

    def _release_live_partition(
        self,
        registration: _Registration,
        partition: str,
        owner: asyncio.Task[Any],
    ) -> None:
        if registration.workers.get(partition) is owner:
            registration.workers.pop(partition, None)
            registration.worker_started_at.pop(partition, None)
            self._rewind_after_partition_release(registration, partition)
        if registration.live and self._active and not self._stopping:
            registration.event.set()

    @staticmethod
    def _rewind_after_partition_release(
        registration: _Registration,
        partition: str,
    ) -> None:
        if partition not in registration.rewind_after_partitions:
            return
        registration.rewind_after_partitions.remove(partition)
        if registration.live:
            registration.rewind_requested = True
            registration.event.set()

    async def run_sync(self, operation: Callable[[], _T]) -> _T:
        """Run blocking store work without abandoning its exact worker future."""

        if self._stopping or self._sync_executor_stopped:
            raise RuntimeError("runtime work executor is stopped")
        loop = asyncio.get_running_loop()
        future = loop.run_in_executor(self._sync_executor, operation)
        self._sync_futures.add(future)
        try:
            # The partition task remains the owner until the exact store operation exits.
            future, cancelled = await land_step(future)
        finally:
            if future.done():
                self._sync_futures.discard(future)
        if cancelled:
            answer_caught_cancellation(future)
        return future.result()

    async def stop(self) -> None:
        stop_task = self._begin_stop()
        await asyncio.shield(stop_task)

    def quiesce(self) -> None:
        """Stop new admission while retaining executor-backed finalization."""

        if self._quiescing or self._stopping:
            return
        self._quiescing = True
        self._active = False
        for registration in self._registrations.values():
            registration.event.set()

    def _begin_stop(self) -> asyncio.Task[None]:
        if self._stop_task is None:
            self._stop_task = asyncio.create_task(
                self._stop_and_join(),
                name="runtime-work-stop",
            )
        return self._stop_task

    def _stop_for_lost_lease(self) -> None:
        if self._quiescing or self._stopping or self._stop_task is not None:
            return
        # Says what was established, not what was inferred from it: the check
        # re-opens the lock file and looks for a record naming this pid, so a
        # false answer means either the lock went away or the record could not
        # be read. Claiming a transition nobody observed sent three release
        # candidates looking for a lock that was never lost.
        logger.error(
            "Runtime work supervisor stopping: re-reading the service lock did "
            "not confirm this process as its holder -- either the lock was "
            "released, or its holder record could not be read"
        )
        self.quiesce()
        if self._on_lease_lost is not None:
            self._on_lease_lost()
            return
        self._begin_stop()

    async def _stop_and_join(self) -> None:
        if self._stopping:
            return
        self.quiesce()
        self._stopping = True
        for handle in self._delayed_wakes.values():
            handle.cancel()
        self._delayed_wakes.clear()
        if self._subscription_id is not None:
            bus.unsubscribe(self._subscription_id)
            self._subscription_id = None
        reconcile = self._reconcile_task
        self._reconcile_task = None
        if reconcile is not None:
            reconcile.cancel()
            await asyncio.gather(reconcile, return_exceptions=True)
        tokens = [registration.token for registration in self._registrations.values()]
        for token in tokens:
            await self.unregister(token)
        remaining = tuple(self._sync_futures)
        if remaining:
            await asyncio.gather(
                *(asyncio.shield(future) for future in remaining),
                return_exceptions=True,
            )
        self._sync_executor_stopped = True
        await asyncio.to_thread(
            self._sync_executor.shutdown,
            wait=True,
            cancel_futures=False,
        )
        self._loop = None

    def _start_registration(self, registration: _Registration) -> None:
        if registration.coordinator is not None:
            return
        registration.coordinator = asyncio.create_task(
            self._coordinate(registration),
            name=f"runtime-work:{registration.token.lane.value}",
        )

    def _on_bus_event(self, event_type: str, _payload: Any) -> None:
        lanes = _EVENT_LANES.get(event_type)
        if not lanes:
            return
        loop = self._loop
        if loop is None or loop.is_closed() or not self._active:
            return
        try:
            loop.call_soon_threadsafe(self._notify_on_loop, lanes)
        except RuntimeError:
            pass

    def _notify_on_loop(
        self,
        lanes: tuple[RuntimeWorkLane, ...] | list[RuntimeWorkLane],
        reset_cursor: bool = False,
    ) -> None:
        if not self._active or self._stopping:
            return
        for lane in lanes:
            registration = self._registrations.get(lane)
            if registration is not None and registration.live:
                if reset_cursor:
                    registration.rewind_requested = True
                registration.event.set()

    def _owns_service_instance(self) -> bool:
        return (
            not self._requires_service_lease
            or runtime.current_process_owns_service_instance()
        )

    async def _coordinate(self, registration: _Registration) -> None:
        while registration.live and not self._stopping:
            await registration.event.wait()
            registration.event.clear()
            pages_remaining = self._scan_continuation_pages
            while pages_remaining > 0:
                if (
                    not registration.live
                    or self._stopping
                    or not self._active
                ):
                    break
                if not self._owns_service_instance():
                    self._stop_for_lost_lease()
                    return
                # Every admitted worker reserves room for its exact partition in
                # the bounded backoff ledger. Once that ledger is full, wait for an
                # existing deadline instead of accepting failures whose identity
                # could not be retained safely.
                capacity = min(
                    self._lane_capacity - len(registration.workers),
                    self._backoff_partition_limit
                    - len(registration.backoff_deadlines)
                    - len(registration.workers),
                )
                if capacity <= 0:
                    break
                occupied = frozenset(
                    (*registration.workers, *registration.backoff_deadlines)
                )
                scan_cursor = (
                    None if registration.rewind_requested else registration.scan_cursor
                )
                registration.rewind_requested = False
                try:
                    registration.scan_started_at = self._monotonic()
                    try:
                        items, has_more = await self.run_sync(
                            partial(
                                registration.handler.scan,
                                limit=min(capacity, self._scan_page_size),
                                occupied=occupied,
                                cursor=scan_cursor,
                            )
                        )
                    finally:
                        registration.scan_started_at = None
                except Exception:
                    logger.exception(
                        "Runtime work scan failed for lane=%s generation=%s",
                        registration.token.lane.value,
                        registration.token.generation,
                    )
                    if registration.live and not self._stopping:
                        await self._retry_wait(self._retry_backoff)
                        if registration.live and self._active and not self._stopping:
                            registration.event.set()
                    break
                if not registration.live or self._stopping:
                    break
                if not self._owns_service_instance():
                    self._stop_for_lost_lease()
                    return
                page_advanced = False
                if items:
                    last_cursor = str(
                        items[-1].cursor_key or items[-1].partition_key or ""
                    ).strip()
                    if last_cursor:
                        page_advanced = last_cursor != scan_cursor
                        registration.scan_cursor = last_cursor
                elif registration.scan_cursor is not None and not has_more:
                    # Reset for the next real wake. Immediately scanning from the
                    # beginning would spin on occupied rows until their owner exits.
                    registration.scan_cursor = None
                for item in items:
                    if item.cursor_only:
                        continue
                    partition = str(item.partition_key or "").strip()
                    if not partition:
                        continue
                    if item.defer_until_partition_release:
                        # Some scans return an occupied partition only to preserve a
                        # wake until its current owner exits. The owner can finish
                        # between the off-loop scan and admission here; in that case
                        # rewind immediately instead of executing a placeholder with
                        # no durable observation.
                        if partition in registration.workers:
                            registration.rewind_after_partitions.add(partition)
                        else:
                            registration.rewind_requested = True
                            registration.event.set()
                        continue
                    if partition in registration.workers:
                        # Row ordering can be finer-grained than the execution
                        # partition. Preserve the skipped durable observation
                        # without polling its still-live owner.
                        registration.rewind_after_partitions.add(partition)
                        continue
                    if partition in registration.backoff_deadlines:
                        continue
                    task = asyncio.create_task(
                        self._run_item(registration, item),
                        name=(
                            f"runtime-work:{registration.token.lane.value}:"
                            f"{partition}"
                        ),
                    )
                    registration.workers[partition] = task
                    registration.worker_started_at[partition] = self._monotonic()
                if not has_more and registration.expired_backoff_rewind_pending:
                    # A retry deadline releases admission capacity but must not pull
                    # the forward cursor back over the same unavailable prefix. Finish
                    # the current durable scan first, then coalesce all expired retries
                    # into one bounded rewind. The store remains the retry inventory.
                    registration.expired_backoff_rewind_pending = False
                    registration.rewind_requested = True
                    if not registration.workers:
                        registration.event.set()
                if not has_more or len(registration.workers) >= self._lane_capacity:
                    break
                if not page_advanced:
                    # A broken/non-keyset handler cannot advance itself. A later
                    # durable wake or reconciliation can retry without a hot loop.
                    break
                # Continue synchronously only while this wake's bounded raw-page
                # budget remains. Partition admission may be coarser than the
                # durable row cursor, so occupied duplicates must not hide a
                # later partition, but they also must not create an unbounded
                # same-tick scan loop.
                pages_remaining -= 1
                if pages_remaining <= 0:
                    self._start_scan_continuation(registration)
                    break

    def _start_scan_continuation(self, registration: _Registration) -> None:
        existing = registration.scan_continuation_task
        if existing is not None and not existing.done():
            return
        registration.scan_continuation_task = asyncio.create_task(
            self._finish_scan_continuation(registration),
            name=(
                f"runtime-work-scan-continuation:"
                f"{registration.token.lane.value}"
            ),
        )

    async def _finish_scan_continuation(
        self,
        registration: _Registration,
    ) -> None:
        try:
            await self._retry_wait(self._retry_backoff)
        finally:
            if registration.scan_continuation_task is asyncio.current_task():
                registration.scan_continuation_task = None
            if registration.live and self._active and not self._stopping:
                registration.event.set()

    async def _run_item(
        self,
        registration: _Registration,
        item: RuntimeWorkItem,
    ) -> None:
        partition = item.partition_key
        should_backoff = False
        try:
            if registration.live and self._owns_service_instance():
                processed = await registration.handler.process(item)
                should_backoff = processed is False
            elif registration.live:
                self._stop_for_lost_lease()
        except asyncio.CancelledError:
            raise
        except Exception:
            should_backoff = True
            logger.exception(
                "Runtime work item failed for lane=%s partition=%s generation=%s",
                registration.token.lane.value,
                partition,
                registration.token.generation,
            )
        finally:
            current = registration.workers.get(partition)
            if current is asyncio.current_task():
                registration.workers.pop(partition, None)
                registration.worker_started_at.pop(partition, None)
                if should_backoff:
                    # Backoff completion already owns the retry rewind. Do not
                    # race it with an immediate scan of the guarded partition.
                    registration.rewind_after_partitions.discard(partition)
                else:
                    self._rewind_after_partition_release(registration, partition)
            if should_backoff and registration.live and not self._stopping:
                self._start_partition_backoff(registration, partition)
            if (
                registration.live
                and self._active
                and not self._stopping
                and (
                    should_backoff
                    or item.rearm_after_process
                    or registration.rewind_requested
                )
            ):
                registration.event.set()

    def _start_partition_backoff(
        self,
        registration: _Registration,
        partition: str,
    ) -> None:
        deadline = self._monotonic() + self._retry_backoff
        if (
            partition not in registration.backoff_deadlines
            and len(registration.backoff_deadlines)
            >= self._backoff_partition_limit
        ):
            raise RuntimeError("runtime work backoff reservation invariant violated")
        registration.backoff_deadlines[partition] = deadline
        existing = registration.backoff_task
        if existing is not None and not existing.done():
            return
        self._start_next_partition_backoff(registration)

    def _start_next_partition_backoff(
        self,
        registration: _Registration,
    ) -> None:
        deadlines = list(registration.backoff_deadlines.values())
        if not deadlines:
            registration.backoff_task = None
            return
        deadline = min(deadlines)
        registration.backoff_task = asyncio.create_task(
            self._finish_partition_backoff(registration, deadline),
            name=f"runtime-work-backoff:{registration.token.lane.value}",
        )

    async def _finish_partition_backoff(
        self,
        registration: _Registration,
        deadline: float,
    ) -> None:
        completed = False
        try:
            await self._retry_wait(max(0.0, deadline - self._monotonic()))
            completed = True
        finally:
            if registration.backoff_task is not asyncio.current_task():
                return
            registration.backoff_task = None
            if completed:
                registration.backoff_deadlines = {
                    partition: partition_deadline
                    for partition, partition_deadline in (
                        registration.backoff_deadlines.items()
                    )
                    if partition_deadline > deadline
                }
                registration.expired_backoff_rewind_pending = True
            if (
                completed
                and registration.live
                and self._active
                and not self._stopping
            ):
                registration.event.set()
            if registration.live and registration.backoff_deadlines:
                self._start_next_partition_backoff(registration)

    async def _reconcile_loop(self) -> None:
        try:
            while self._active and not self._stopping:
                await self._reconcile_wait(self._reconcile_interval)
                if not self._owns_service_instance():
                    self._stop_for_lost_lease()
                    return
                self._report_overdue_workers()
                self._notify_on_loop(tuple(self._registrations), True)
        except asyncio.CancelledError:
            raise

    def _report_overdue_workers(self) -> None:
        now = self._monotonic()
        for lane, registration in tuple(self._registrations.items()):
            if not registration.live:
                continue
            if registration.scan_started_at is not None:
                elapsed = now - registration.scan_started_at
                if elapsed >= self._overdue_seconds:
                    logger.warning(
                        "Runtime work item overdue lane=%s partition=%s "
                        "generation=%s elapsed=%.3fs",
                        lane.value,
                        "<scan>",
                        registration.token.generation,
                        elapsed,
                    )
            for partition, started_at in tuple(registration.worker_started_at.items()):
                elapsed = now - started_at
                if elapsed < self._overdue_seconds:
                    continue
                logger.warning(
                    "Runtime work item overdue lane=%s partition=%s generation=%s "
                    "elapsed=%.3fs",
                    lane.value,
                    partition,
                    registration.token.generation,
                    elapsed,
                )