Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

38.8 KB
Raw Blame
"""Tests for workflows/resumer.py."""
from __future__ import annotations

import asyncio
import json
from datetime import UTC, datetime, timedelta
from pathlib import Path

import pytest

from openexecutive.alerts import store as alert_store
from openexecutive.departments import store as dept_store
from openexecutive.memory import episodic
from openexecutive.people import store as people_store
from openexecutive.workflows import persistence as wf_persistence
from openexecutive.workflows.resumer import _handle_timeout, _tick, apply_resolution
from openexecutive.workflows.wait_for_human import WaitForHumanResolution


@pytest.fixture(autouse=True)
def _isolated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
    db = tmp_path / "test.db"
    monkeypatch.setattr(episodic, "DB_PATH", db)
    monkeypatch.setattr(wf_persistence, "DB_PATH", db)
    monkeypatch.setattr(alert_store, "DB_PATH", db)
    monkeypatch.setattr(dept_store, "DB_PATH", db)
    monkeypatch.setattr(people_store, "DB_PATH", db)

    episodic.initialize_db(db)
    wf_persistence.initialize_runs_db(db)
    alert_store.initialize_db(db)
    dept_store.initialize_db(db)
    people_store.initialize_db(db)
    yield


def _seed_awaiting(
    run_id: str,
    person_id: int,
    on_timeout: str = "escalate",
    awaiting_until: datetime | None = None,
    *,
    department: str = "",
    resume_state: str | None = None,
    workflow_name: str = "test_wf",
    inputs: dict | None = None,
) -> None:
    wf_persistence.create_run(run_id, workflow_name, "Test run", inputs or {})
    state = json.dumps({
        "on_timeout": on_timeout,
        "channel": "slack",
        "channel_ref": "U123",
        "expected_reply_shape": "approve_reject",
        "question": "Please approve.",
        "department": department,
    })
    until = awaiting_until or datetime.now(UTC) - timedelta(minutes=1)  # already expired
    wf_persistence.save_checkpoint(
        run_id, state, person_id, until, resume_state_json=resume_state
    )


# ---------------------------------------------------------------------------
# apply_resolution
# ---------------------------------------------------------------------------

def test_apply_resolution_marks_resolved() -> None:
    _seed_awaiting("run-1", person_id=5)
    resolution = WaitForHumanResolution(
        run_id="run-1",
        reply_text="approved",
        source_channel="slack",
        source_message_id="msg-1",
        parsed_decision={"decision": "approve", "note": ""},
        person_id=5,
    )
    result = asyncio.run(apply_resolution("run-1", resolution))
    assert result is True

    run = wf_persistence.get_run("run-1")
    assert run is not None
    assert run["status"] == "resolved"
    loaded_res = json.loads(run["resolution_json"])
    assert loaded_res["parsed_decision"]["decision"] == "approve"


def test_apply_resolution_is_idempotent() -> None:
    _seed_awaiting("run-2", person_id=3)
    resolution = WaitForHumanResolution(
        run_id="run-2",
        reply_text="ok",
        source_channel="telegram",
        parsed_decision={"decision": "approve", "note": ""},
        person_id=3,
    )
    first = asyncio.run(apply_resolution("run-2", resolution))
    second = asyncio.run(apply_resolution("run-2", resolution))
    assert first is True
    assert second is False  # run is no longer awaiting_human


def test_apply_resolution_writes_audit_log() -> None:
    from openexecutive.audit.logger import AuditLogger, set_audit_logger
    db = episodic.DB_PATH  # already monkeypatched to tmp_path in autouse fixture
    audit_logger = AuditLogger(db)
    set_audit_logger(audit_logger)

    _seed_awaiting("run-3", person_id=7)
    resolution = WaitForHumanResolution(
        run_id="run-3",
        reply_text="yes",
        source_channel="slack",
        parsed_decision={"decision": "approve", "note": ""},
        person_id=7,
    )
    asyncio.run(apply_resolution("run-3", resolution))

    events = audit_logger.query(event_type="human_resolution", limit=5)
    assert any("run-3" in str(e.summary) for e in events)


# ---------------------------------------------------------------------------
# Timeout handling
# ---------------------------------------------------------------------------

def test_timeout_escalate_creates_alert() -> None:
    dept_store.seed_default_departments()
    principal_id = people_store.upsert_person(full_name="Founder", is_principal=True)
    from openexecutive.people.models import AuthorityScope
    people_store.set_authority_scope(principal_id, [AuthorityScope.WILDCARD])

    _seed_awaiting("run-esc", person_id=principal_id, on_timeout="escalate", department="finance")
    run = wf_persistence.list_awaiting_runs()[0]
    asyncio.run(_handle_timeout(run, datetime.now(UTC)))

    run_after = wf_persistence.get_run("run-esc")
    assert run_after is not None
    assert run_after["status"] == "timed_out"


def test_timeout_fail_sets_error_status() -> None:
    _seed_awaiting("run-fail", person_id=1, on_timeout="fail")
    run = wf_persistence.list_awaiting_runs()[0]
    asyncio.run(_handle_timeout(run, datetime.now(UTC)))

    run_after = wf_persistence.get_run("run-fail")
    assert run_after is not None
    assert run_after["status"] == "error"


def test_timeout_auto_proceed_marks_resolved() -> None:
    _seed_awaiting("run-auto", person_id=2, on_timeout="auto_proceed")
    run = wf_persistence.list_awaiting_runs()[0]
    asyncio.run(_handle_timeout(run, datetime.now(UTC)))

    run_after = wf_persistence.get_run("run-auto")
    assert run_after is not None
    assert run_after["status"] == "resolved"
    loaded_res = json.loads(run_after["resolution_json"])
    assert loaded_res["parsed_decision"]["decision"] == "auto_proceed"


# ---------------------------------------------------------------------------
# _tick — poll cycle
# ---------------------------------------------------------------------------

def test_tick_processes_expired_runs() -> None:
    past = datetime.now(UTC) - timedelta(hours=1)
    future = datetime.now(UTC) + timedelta(hours=24)

    _seed_awaiting("expired", person_id=1, on_timeout="fail", awaiting_until=past)
    _seed_awaiting("not-yet", person_id=1, on_timeout="fail", awaiting_until=future)

    asyncio.run(_tick(datetime.now(UTC)))

    assert wf_persistence.get_run("expired")["status"] == "error"
    assert wf_persistence.get_run("not-yet")["status"] == "awaiting_human"


def test_tick_no_runs_is_no_op() -> None:
    # No awaiting runs — must not raise.
    asyncio.run(_tick(datetime.now(UTC)))


# ---------------------------------------------------------------------------
# Acknowledgement wording (#136)
# ---------------------------------------------------------------------------


def test_acknowledgement_names_the_decision_and_the_limit(tmp_path: Path) -> None:
    """"Got it — your response has been recorded." implied the workflow was
    now proceeding. It is not: apply_resolution stores the decision and
    releases the run, and nothing downstream runs (generator resume is
    unimplemented). Saying otherwise is the same class of overclaim #136 was
    filed for."""
    from openexecutive.workflows.resumer import resolution_acknowledgement

    db = tmp_path / "runs.db"
    wf_persistence.create_run("run-1", "vendor_review", "Vendor Review", {}, db_path=db)

    text = resolution_acknowledgement(
        "run-1",
        WaitForHumanResolution(
            run_id="run-1",
            reply_text="yes",
            source_channel="slack",
            parsed_decision={"decision": "approve", "note": ""},
            person_id=7,
        ),
        db_path=db,
    )

    assert text.startswith("Approved")
    assert "Vendor Review" in text
    assert "doesn't pick up from here on its own" in text


def test_acknowledgement_reflects_a_rejection(tmp_path: Path) -> None:
    from openexecutive.workflows.resumer import resolution_acknowledgement

    db = tmp_path / "runs.db"
    wf_persistence.create_run("run-1", "vendor_review", "Vendor Review", {}, db_path=db)

    text = resolution_acknowledgement(
        "run-1",
        WaitForHumanResolution(
            run_id="run-1",
            reply_text="no",
            source_channel="slack",
            parsed_decision={"decision": "reject", "note": "too expensive"},
            person_id=7,
        ),
        db_path=db,
    )

    assert text.startswith("Declined")


def test_acknowledgement_survives_a_missing_run(tmp_path: Path) -> None:
    from openexecutive.workflows.resumer import resolution_acknowledgement

    text = resolution_acknowledgement(
        "nope",
        WaitForHumanResolution(
            run_id="nope",
            reply_text="yes",
            source_channel="slack",
            parsed_decision={"decision": "approve"},
            person_id=7,
        ),
        db_path=tmp_path / "missing.db",
    )

    assert text.startswith("Approved")


# ---------------------------------------------------------------------------
# Resume — the background executor
# ---------------------------------------------------------------------------

def _resume_state_json(fingerprint: str = "") -> str:
    """A payload matching the definition `_real_workflow()` builds.

    Most of these tests replace `resume()` outright, so the fingerprint is
    never checked — but the one that doesn't would fail on a hardcoded value,
    and a literal here would silently rot the moment that definition changes.
    """
    return json.dumps({
        "version": 1,
        "engine": "dynamic",
        "workflow_name": "weekly_watch",
        "gate_step_id": "gate",
        "gate_step_index": 1,
        "steps_fingerprint": fingerprint,
        "outputs": {"research": ["Research", "prior output"]},
    })


_RESUME_STATE = _resume_state_json()


def _resolved(
    run_id: str = "res-1",
    decision: str = "approve",
    *,
    resume_state: str | None = _RESUME_STATE,
    workflow_name: str = "weekly_watch",
    inputs: dict | None = None,
) -> None:
    """A run sitting at `resolved` with (by default) a resume payload."""
    _seed_awaiting(
        run_id, 7, awaiting_until=datetime.now(UTC) + timedelta(hours=24),
        resume_state=resume_state, workflow_name=workflow_name,
        inputs=inputs if inputs is not None else {"topic": "pricing"},
    )
    wf_persistence.store_resolution(
        run_id,
        json.dumps({
            "run_id": run_id, "reply_text": "yes", "source_channel": "slack",
            "parsed_decision": {"decision": decision, "note": "fine"},
            "person_id": 7,
        }),
    )


def _real_workflow(events: list | None = None):  # noqa: ANN201
    """A genuine DynamicWorkflow whose `resume` is replaced.

    Deliberately not a duck-typed stub: `_execute_resume` guards on
    `isinstance(workflow, DynamicWorkflow)` to catch a dynamic name later
    shadowed by a built-in, and a fake would have to neuter that guard to be
    accepted — testing a path production never takes. This keeps the real
    type, the real input model, and records what resume() was handed.
    """
    from openexecutive.workflows.base import WorkflowEvent
    from openexecutive.workflows.dynamic import DynamicWorkflow
    from openexecutive.workflows.dynamic_models import DynamicWorkflowDef

    defn = DynamicWorkflowDef.model_validate({
        "name": "weekly_watch",
        "title": "Weekly Watch",
        "input_fields": [{"name": "topic", "label": "Topic", "required": False}],
        "steps": [
            {"kind": "specialist", "id": "research", "title": "Research",
             "specialist": "cso", "goal": "Analyze."},
            {"kind": "approval_gate", "id": "gate", "title": "Approve",
             "person_id": 7, "question": "OK?"},
            {"kind": "synthesis", "id": "assemble", "title": "Assemble"},
        ],
    })
    wf = DynamicWorkflow(defn)
    calls: list[dict] = []

    async def _resume(*, inputs, state, resolution, store):  # noqa: ANN001, ANN202
        calls.append({"state": state, "resolution": resolution, "inputs": inputs})
        for ev in (events if events is not None else [
            WorkflowEvent(type="artifact", content="# Final artifact")
        ]):
            yield ev

    wf.resume = _resume  # type: ignore[method-assign]
    wf.calls = calls  # type: ignore[attr-defined]
    return wf


def _install_stub(monkeypatch: pytest.MonkeyPatch, stub) -> None:  # noqa: ANN001
    """Point the executor at this workflow, with a no-op store and audit."""
    import openexecutive.audit as audit
    import openexecutive.knowledge.store as knowledge_store
    import openexecutive.workflows as wf_pkg

    monkeypatch.setattr(wf_pkg, "get_workflow", lambda name: stub)
    monkeypatch.setattr(knowledge_store, "ChromaDBStore", lambda *a, **k: object())
    monkeypatch.setattr(audit, "log_event", lambda *a, **k: None)


def test_resume_executes_remaining_steps_and_completes(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The headline behaviour: a resolved run's remaining steps run and the
    artifact lands. Before this, the run simply stopped at `resolved`."""
    from openexecutive.workflows.resumer import _process_resumable

    _resolved()
    stub = _real_workflow()
    _install_stub(monkeypatch, stub)

    executed = asyncio.run(_process_resumable(datetime.now(UTC)))

    assert executed == 1
    run = wf_persistence.get_run("res-1")
    assert run["status"] == "done"
    assert run["artifact"] == "# Final artifact"
    # The payload is dropped so a finished run can never be re-claimed.
    assert run["resume_state_json"] is None
    # The engine was handed the recorded gate and the human's decision.
    assert stub.calls[0]["state"].gate_step_id == "gate"
    assert stub.calls[0]["state"].gate_step_index == 1
    assert stub.calls[0]["resolution"].parsed_decision["decision"] == "approve"


def test_resume_failure_is_recorded_as_a_failed_run(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A declined gate surfaces as an error event; the run must end at `error`
    rather than hanging in `running` with the claim still on it."""
    from openexecutive.workflows.base import WorkflowEvent
    from openexecutive.workflows.resumer import _process_resumable

    _resolved(decision="reject")
    _install_stub(monkeypatch, _real_workflow(
        [WorkflowEvent(type="error", message="Declined at the 'Approve' gate: fine")]
    ))

    asyncio.run(_process_resumable(datetime.now(UTC)))

    run = wf_persistence.get_run("res-1")
    assert run["status"] == "error"
    assert "Declined" in run["error"]
    assert run["resume_state_json"] is None


def test_pause_only_runs_are_never_claimed(monkeypatch: pytest.MonkeyPatch) -> None:
    """A gate with no payload is finished at `resolved`. Claiming it would
    flip a completed run back to `running` with nothing to execute."""
    from openexecutive.workflows.resumer import _process_resumable

    _resolved(resume_state=None)
    _install_stub(monkeypatch, _real_workflow())

    assert asyncio.run(_process_resumable(datetime.now(UTC))) == 0
    assert wf_persistence.get_run("res-1")["status"] == "resolved"


def test_a_claimed_run_is_not_executed_twice(monkeypatch: pytest.MonkeyPatch) -> None:
    """The immediate kick and the poll loop both reach the executor; the
    atomic claim is what stops a run being paid for twice."""
    from openexecutive.workflows.resumer import _process_resumable

    _resolved()
    stub = _real_workflow()
    _install_stub(monkeypatch, stub)

    asyncio.run(_process_resumable(datetime.now(UTC)))
    asyncio.run(_process_resumable(datetime.now(UTC)))

    assert len(stub.calls) == 1


def test_second_gate_recheckpoints_instead_of_completing(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A workflow with two gates pauses again mid-resume. The run must go back
    to `awaiting_human` with the NEW payload and the OLD resolution cleared —
    otherwise the executor would re-claim it on gate 1's answer."""
    from openexecutive.workflows.resumer import _process_resumable
    from openexecutive.workflows.wait_for_human import (
        WaitForHumanEvent,
        WorkflowResumeState,
    )

    _resolved()
    second = WaitForHumanEvent(
        person_id=7, question="Ship it?",
        resume_state=WorkflowResumeState(
            workflow_name="weekly_watch", gate_step_id="gate2",
            gate_step_index=3, next_step_index=4, outputs={},
        ),
    )
    _install_stub(monkeypatch, _real_workflow([second]))

    async def _deliver(event, **_):  # noqa: ANN001, ANN202
        return event.model_copy(update={"delivery": "sent"}), "sent"

    monkeypatch.setattr(
        "openexecutive.workflows.gate_delivery.deliver_gate_question", _deliver
    )

    asyncio.run(_process_resumable(datetime.now(UTC)))

    run = wf_persistence.get_run("res-1")
    assert run["status"] == "awaiting_human"
    assert json.loads(run["resume_state_json"])["gate_step_id"] == "gate2"
    assert run["resolution_json"] is None
    assert run["artifact"] is None


def test_resume_fails_when_the_workflow_was_deleted(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A custom workflow can be deleted while a run sits at its gate."""
    import openexecutive.workflows as wf_pkg
    from openexecutive.workflows.resumer import _process_resumable

    _resolved()

    def _missing(name: str):  # noqa: ANN202
        raise KeyError(name)

    monkeypatch.setattr(wf_pkg, "get_workflow", _missing)
    monkeypatch.setattr(
        "openexecutive.workflows.dynamic_store.get_definition", lambda name: None
    )

    asyncio.run(_process_resumable(datetime.now(UTC)))

    run = wf_persistence.get_run("res-1")
    assert run["status"] == "error"
    assert "deleted" in run["error"]
    assert run["resume_state_json"] is None


def test_resume_says_deactivated_when_the_definition_still_exists(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Deleted and switched-off need different things from the reader."""
    import openexecutive.workflows as wf_pkg
    from openexecutive.workflows.resumer import _process_resumable

    _resolved()

    def _missing(name: str):  # noqa: ANN202
        raise KeyError(name)

    monkeypatch.setattr(wf_pkg, "get_workflow", _missing)
    monkeypatch.setattr(
        "openexecutive.workflows.dynamic_store.get_definition",
        lambda name: object(),
    )

    asyncio.run(_process_resumable(datetime.now(UTC)))

    assert "deactivated" in wf_persistence.get_run("res-1")["error"]


def test_unreadable_payload_fails_the_run_rather_than_the_sweep(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from openexecutive.workflows.resumer import _process_resumable

    _resolved(resume_state="{not json")
    _install_stub(monkeypatch, _real_workflow())

    asyncio.run(_process_resumable(datetime.now(UTC)))

    run = wf_persistence.get_run("res-1")
    assert run["status"] == "error"
    assert "could not be read" in run["error"]


def test_auto_proceed_timeout_executes_in_the_same_tick(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """`on_timeout='auto_proceed'` was inert: it synthesized a resolution and
    nothing ran. `_tick` calls the executor AFTER the timeout loop, which is
    what finally makes it proceed — with no special-casing anywhere."""
    _seed_awaiting(
        "auto-1", 7, on_timeout="auto_proceed",
        resume_state=_RESUME_STATE, workflow_name="weekly_watch",
        inputs={"topic": "pricing"},
    )
    stub = _real_workflow()
    _install_stub(monkeypatch, stub)

    asyncio.run(_tick(datetime.now(UTC)))

    run = wf_persistence.get_run("auto-1")
    assert run["status"] == "done"
    assert run["artifact"] == "# Final artifact"
    assert stub.calls[0]["resolution"].parsed_decision["decision"] == "auto_proceed"


def test_escalate_timeout_drops_the_resume_payload() -> None:
    """`timed_out` is terminal — nobody answered, so the payload will never be
    replayed and must not keep the run on the resumable queue."""
    _seed_awaiting("esc-1", 7, on_timeout="escalate", resume_state=_RESUME_STATE)

    asyncio.run(_tick(datetime.now(UTC)))

    run = wf_persistence.get_run("esc-1")
    assert run["status"] == "timed_out"
    assert run["resume_state_json"] is None


def test_a_stranded_resume_is_requeued_and_retried(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Crash recovery: a worker that died after claiming leaves the run in
    `running`, which no other sweep looks at."""
    from openexecutive.workflows.resumer import _RESUME_STALE_AFTER, _process_resumable

    _resolved()
    wf_persistence.claim_run_for_resume("res-1")  # simulate the dead worker
    assert wf_persistence.get_run("res-1")["status"] == "running"

    stub = _real_workflow()
    _install_stub(monkeypatch, stub)
    later = datetime.now(UTC) + _RESUME_STALE_AFTER + timedelta(minutes=1)

    assert asyncio.run(_process_resumable(later)) == 1
    assert wf_persistence.get_run("res-1")["status"] == "done"


def test_acknowledgement_says_the_run_is_resuming() -> None:
    """#143 shipped wording promising the opposite; a resumable gate must not
    tell the approver to come back and ask for the next step themselves."""
    from openexecutive.workflows.resumer import resolution_acknowledgement

    _resolved()
    ack = resolution_acknowledgement(
        "res-1",
        WaitForHumanResolution(
            run_id="res-1", reply_text="yes", source_channel="slack",
            parsed_decision={"decision": "approve"}, person_id=7,
        ),
    )
    assert "Approved" in ack
    assert "picking up" in ack
    assert "doesn't pick up from here on its own" not in ack


def test_acknowledgement_says_the_run_stops_on_a_decline() -> None:
    from openexecutive.workflows.resumer import resolution_acknowledgement

    _resolved(decision="reject")
    ack = resolution_acknowledgement(
        "res-1",
        WaitForHumanResolution(
            run_id="res-1", reply_text="no", source_channel="slack",
            parsed_decision={"decision": "reject"}, person_id=7,
        ),
    )
    assert "Declined" in ack
    assert "stops there" in ack


def test_acknowledgement_keeps_the_old_wording_for_a_pause_only_gate() -> None:
    """Nothing continues a gate with no payload, and saying otherwise would be
    the same lie in the other direction."""
    from openexecutive.workflows.resumer import resolution_acknowledgement

    _resolved(resume_state=None)
    ack = resolution_acknowledgement(
        "res-1",
        WaitForHumanResolution(
            run_id="res-1", reply_text="yes", source_channel="slack",
            parsed_decision={"decision": "approve"}, person_id=7,
        ),
    )
    assert "doesn't pick up from here on its own" in ack


def test_resume_refuses_a_workflow_that_is_no_longer_dynamic(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """`get_workflow` checks the built-in registry first, so a custom
    workflow's name can be shadowed by a built-in shipped later. Running the
    built-in's steps against this run's payload would be nonsense."""
    import openexecutive.audit as audit
    import openexecutive.knowledge.store as knowledge_store
    import openexecutive.workflows as wf_pkg
    from openexecutive.workflows import WORKFLOW_REGISTRY
    from openexecutive.workflows.resumer import _process_resumable

    _resolved()
    builtin = WORKFLOW_REGISTRY["board_prep"]
    monkeypatch.setattr(wf_pkg, "get_workflow", lambda name: builtin)
    monkeypatch.setattr(knowledge_store, "ChromaDBStore", lambda *a, **k: object())
    monkeypatch.setattr(audit, "log_event", lambda *a, **k: None)

    asyncio.run(_process_resumable(datetime.now(UTC)))

    run = wf_persistence.get_run("res-1")
    assert run["status"] == "error"
    assert "no longer a custom workflow" in run["error"]


def test_a_crash_mid_resume_fails_the_run_on_both_paths(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The kick and the poll must agree. An earlier version had the poll fail
    the run while the kick only logged, which left a crashed run sitting in
    `running` waiting on a 30-minute sweep — a different outcome for the same
    failure depending on which entry point happened to win the claim."""
    from openexecutive.workflows.resumer import _process_resumable

    _resolved()
    wf = _real_workflow()

    async def _boom(*a, **k):  # noqa: ANN002, ANN003, ANN202
        raise RuntimeError("the model fell over")
        yield  # pragma: no cover - makes this an async generator

    wf.resume = _boom  # type: ignore[method-assign]
    _install_stub(monkeypatch, wf)

    asyncio.run(_process_resumable(datetime.now(UTC)))

    run = wf_persistence.get_run("res-1")
    assert run["status"] == "error"
    # Off the resumable queue, so it is not retried forever.
    assert run["resume_state_json"] is None


def test_a_gate_as_the_final_step_fails_rather_than_hanging(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """`validate_definition` requires synthesis last, but a definition stored
    before that rule can still reach the engine. Resuming past a trailing gate
    runs no steps and produces no artifact; the run must end at `error` with a
    legible reason instead of silently reporting success."""
    from openexecutive.workflows.resumer import _process_resumable

    _resolved()
    # next_step_index == len(steps): the loop body never executes.
    _install_stub(monkeypatch, _real_workflow(events=[]))

    asyncio.run(_process_resumable(datetime.now(UTC)))

    run = wf_persistence.get_run("res-1")
    assert run["status"] == "error"
    assert "without producing an artifact" in run["error"]


def test_a_run_past_its_retry_budget_is_ended(monkeypatch: pytest.MonkeyPatch) -> None:
    """Exceeding the cap must be terminal, not a disappearance. The row is
    `running` with a payload, so it is in neither queue — without this it shows
    as in-progress on /jobs forever, with nothing coming back for it."""
    from openexecutive.workflows.resumer import (
        _MAX_RESUME_ATTEMPTS,
        _RESUME_STALE_AFTER,
        _process_resumable,
    )

    _resolved()
    for _ in range(_MAX_RESUME_ATTEMPTS):
        wf_persistence.claim_run_for_resume("res-1")
        wf_persistence.requeue_run_for_resume("res-1")
    wf_persistence.claim_run_for_resume("res-1")  # the claim that never returns
    assert wf_persistence.get_run("res-1")["status"] == "running"

    _install_stub(monkeypatch, _real_workflow())
    later = datetime.now(UTC) + _RESUME_STALE_AFTER + timedelta(minutes=1)
    asyncio.run(_process_resumable(later))

    run = wf_persistence.get_run("res-1")
    assert run["status"] == "error"
    assert "attempts" in run["error"]
    assert run["resume_state_json"] is None


def test_a_reply_arriving_mid_timeout_is_not_discarded() -> None:
    """`_tick` lists expired rows, then acts on them. A reply landing in that
    gap already moved the run to `resolved`; every write in the escalate branch
    is unguarded, so continuing would strip the payload off a run that is
    queued to resume — approval recorded, never executed."""
    from openexecutive.workflows.resumer import _handle_timeout

    _seed_awaiting("race-1", 7, on_timeout="escalate", resume_state=_RESUME_STATE)
    run = dict(wf_persistence.list_awaiting_runs()[0])  # as _tick would see it

    # The human answers before _handle_timeout gets there.
    wf_persistence.store_resolution("race-1", json.dumps({"decision": "approve"}))

    asyncio.run(_handle_timeout(run, datetime.now(UTC)))

    row = wf_persistence.get_run("race-1")
    assert row["status"] == "resolved", "the human's answer must stand"
    assert row["resume_state_json"] is not None, "and it must still be resumable"


def test_a_reply_arriving_mid_fail_timeout_is_not_discarded() -> None:
    """Same race, the `on_timeout='fail'` branch — `fail_run` has no status
    guard, so it would overwrite the resolution with `error`."""
    from openexecutive.workflows.resumer import _handle_timeout

    _seed_awaiting("race-2", 7, on_timeout="fail", resume_state=_RESUME_STATE)
    run = dict(wf_persistence.list_awaiting_runs()[0])
    wf_persistence.store_resolution("race-2", json.dumps({"decision": "approve"}))

    asyncio.run(_handle_timeout(run, datetime.now(UTC)))

    assert wf_persistence.get_run("race-2")["status"] == "resolved"


def test_a_second_gate_restores_the_retry_budget(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """End to end: reaching gate 2 resets `resume_attempts`, so a multi-gate
    workflow does not arrive at its last gate with no crash-recovery budget
    left having never failed at anything."""
    from openexecutive.workflows.resumer import _process_resumable
    from openexecutive.workflows.wait_for_human import (
        WaitForHumanEvent,
        WorkflowResumeState,
    )

    _resolved()
    second = WaitForHumanEvent(
        person_id=7, question="Ship it?",
        resume_state=WorkflowResumeState(
            workflow_name="weekly_watch", gate_step_id="gate2",
            gate_step_index=3, steps_fingerprint="abc", outputs={},
        ),
    )
    _install_stub(monkeypatch, _real_workflow([second]))

    async def _deliver(event, **_):  # noqa: ANN001, ANN202
        return event.model_copy(update={"delivery": "sent"}), "sent"

    monkeypatch.setattr(
        "openexecutive.workflows.gate_delivery.deliver_gate_question", _deliver
    )

    asyncio.run(_process_resumable(datetime.now(UTC)))

    import sqlite3
    conn = sqlite3.connect(str(wf_persistence.DB_PATH))
    attempts = conn.execute(
        "SELECT resume_attempts FROM workflow_runs WHERE run_id = 'res-1'"
    ).fetchone()[0]
    conn.close()
    assert attempts == 0


def test_a_second_gate_pause_is_not_counted_as_a_completion(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """`executed` is logged as "executed N resumed run(s)"; a run that merely
    parked again has not been executed."""
    from openexecutive.workflows.resumer import _process_resumable
    from openexecutive.workflows.wait_for_human import (
        WaitForHumanEvent,
        WorkflowResumeState,
    )

    _resolved()
    second = WaitForHumanEvent(
        person_id=7, question="Ship it?",
        resume_state=WorkflowResumeState(
            workflow_name="weekly_watch", gate_step_id="gate2",
            gate_step_index=3, steps_fingerprint="abc", outputs={},
        ),
    )
    _install_stub(monkeypatch, _real_workflow([second]))

    async def _deliver(event, **_):  # noqa: ANN001, ANN202
        return event.model_copy(update={"delivery": "sent"}), "sent"

    monkeypatch.setattr(
        "openexecutive.workflows.gate_delivery.deliver_gate_question", _deliver
    )

    assert asyncio.run(_process_resumable(datetime.now(UTC))) == 0


# ---------------------------------------------------------------------------
# Heartbeat — a long resume (workflow action steps) must never look dead
# ---------------------------------------------------------------------------


def test_touch_resume_claim_only_for_the_live_claim() -> None:
    _resolved()
    token = wf_persistence.claim_run_for_resume("res-1")
    assert token is not None
    before = wf_persistence.get_run("res-1")["resumed_at"]
    assert wf_persistence.touch_resume_claim("res-1", token) is True
    assert wf_persistence.get_run("res-1")["resumed_at"] >= before
    assert wf_persistence.touch_resume_claim("res-1", "not-the-claim") is False


def test_heartbeat_keeps_a_long_resume_off_the_stale_list(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Events refresh `resumed_at`, so the stale sweep never requeues (and
    replays the side effects of) a resume that is still working."""
    from openexecutive.workflows import resumer
    from openexecutive.workflows.base import WorkflowEvent

    monkeypatch.setattr(resumer, "_RESUME_HEARTBEAT_EVERY", timedelta(0))
    _resolved()
    wf = _real_workflow()
    seen_stale: list[list[str]] = []

    async def _resume(*, inputs, state, resolution, store):  # noqa: ANN001, ANN202
        # Pretend this worker has been busy for longer than the stale window.
        with wf_persistence._get_conn(wf_persistence._resolve(None)) as conn:
            conn.execute(
                "UPDATE workflow_runs SET resumed_at = ? WHERE run_id = 'res-1'",
                ((datetime.now(UTC) - timedelta(days=1)).isoformat(),),
            )
        yield WorkflowEvent(type="progress", step_id="act", summary="Using a tool…")
        seen_stale.append(
            wf_persistence.list_stale_resuming_runs(
                datetime.now(UTC) - resumer._RESUME_STALE_AFTER, 3
            )
        )
        yield WorkflowEvent(type="artifact", content="# Done")

    wf.resume = _resume  # type: ignore[method-assign]
    _install_stub(monkeypatch, wf)
    asyncio.run(resumer._process_resumable(datetime.now(UTC)))
    assert seen_stale == [[]]
    assert wf_persistence.get_run("res-1")["status"] == "done"


def test_a_superseded_resume_stops_before_acting_again(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """If the sweep handed the run to another worker, this one must stop at
    its next event — not carry on sending/writing a second time."""
    from openexecutive.workflows import resumer
    from openexecutive.workflows.base import WorkflowEvent

    monkeypatch.setattr(resumer, "_RESUME_HEARTBEAT_EVERY", timedelta(0))
    _resolved()
    wf = _real_workflow()
    reached: list[str] = []

    async def _resume(*, inputs, state, resolution, store):  # noqa: ANN001, ANN202
        yield WorkflowEvent(type="progress", step_id="act", summary="Using x…")
        wf_persistence.requeue_run_for_resume("res-1")  # the sweep gave up on us…
        wf_persistence.claim_run_for_resume("res-1")  # …and another worker took it
        yield WorkflowEvent(type="progress", step_id="act", summary="Using y…")
        reached.append("kept going")
        yield WorkflowEvent(type="artifact", content="# must not land")

    wf.resume = _resume  # type: ignore[method-assign]
    _install_stub(monkeypatch, wf)
    assert asyncio.run(resumer._process_resumable(datetime.now(UTC))) == 0
    assert reached == []
    run = wf_persistence.get_run("res-1")
    assert run["status"] == "running" and run["artifact"] is None


def test_a_busy_database_on_heartbeat_does_not_kill_the_resume(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    import sqlite3

    from openexecutive.workflows import resumer
    from openexecutive.workflows.base import WorkflowEvent

    monkeypatch.setattr(resumer, "_RESUME_HEARTBEAT_EVERY", timedelta(0))

    def _locked(*a, **k):  # noqa: ANN002, ANN003, ANN202
        raise sqlite3.OperationalError("database is locked")

    monkeypatch.setattr(wf_persistence, "touch_resume_claim", _locked)
    _resolved()
    wf = _real_workflow(
        [
            WorkflowEvent(type="progress", step_id="act", summary="Using x…"),
            WorkflowEvent(type="artifact", content="# Done"),
        ]
    )
    _install_stub(monkeypatch, wf)
    assert asyncio.run(resumer._process_resumable(datetime.now(UTC))) == 1
    assert wf_persistence.get_run("res-1")["status"] == "done"


def test_requeue_with_cutoff_spares_a_freshly_heartbeated_run() -> None:
    """The sweep reads stale ids, then requeues; a heartbeat landing in
    between must win, or a live worker's run is handed to a second one."""
    _resolved()
    token = wf_persistence.claim_run_for_resume("res-1")
    assert token is not None
    cutoff = datetime.now(UTC) - timedelta(minutes=90)
    wf_persistence.touch_resume_claim("res-1", token)  # fresh heartbeat
    assert wf_persistence.requeue_run_for_resume("res-1", stale_before=cutoff) is False
    assert wf_persistence.get_run("res-1")["resume_claim"] == token
    # A genuinely stale claim is still requeued.
    assert wf_persistence.requeue_run_for_resume(
        "res-1", stale_before=datetime.now(UTC) + timedelta(minutes=1)
    ) is True


def _scheduled_resume_state() -> str:
    state = json.loads(_RESUME_STATE)
    state["deliver_to_person_id"] = 11
    return json.dumps(state)


def _capture_dms(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
    sent: list[dict] = []

    async def _send(payload: dict) -> str:
        sent.append(payload)
        return "{}"

    async def _message(**kw: str) -> str:
        return f"About {kw['title']}: {kw['artifact']}"

    monkeypatch.setattr("openexecutive.orchestrator.schedule_tools.handle_message_person", _send)
    monkeypatch.setattr("openexecutive.workflows.run_message.message_for_run", _message)
    return sent


def test_resumed_scheduled_run_dms_its_artifact(monkeypatch: pytest.MonkeyPatch) -> None:
    """A scheduled run that paused (held writes) completes in the resumer, which
    makes the delivery the scheduler would have made."""
    from openexecutive.workflows.resumer import _process_resumable

    sent = _capture_dms(monkeypatch)
    _resolved(resume_state=_scheduled_resume_state())
    _install_stub(monkeypatch, _real_workflow())
    assert asyncio.run(_process_resumable(datetime.now(UTC))) == 1
    assert len(sent) == 1 and sent[0]["person_id"] == 11
    assert sent[0]["text"].endswith(": # Final artifact")

    # A run with no scheduled recipient delivers nothing.
    sent.clear()
    _resolved(run_id="res-2")
    _install_stub(monkeypatch, _real_workflow())
    asyncio.run(_process_resumable(datetime.now(UTC)))
    assert sent == []


def test_a_later_pause_keeps_the_scheduled_recipient(monkeypatch: pytest.MonkeyPatch) -> None:
    from openexecutive.workflows.resumer import _process_resumable
    from openexecutive.workflows.wait_for_human import WaitForHumanEvent, WorkflowResumeState

    sent = _capture_dms(monkeypatch)
    later = WaitForHumanEvent(
        person_id=7, question="Again?",
        resume_state=WorkflowResumeState(
            workflow_name="weekly_watch", gate_step_id="gate", gate_step_index=1,
            steps_fingerprint="x",
        ),
    )
    checkpointed: list[WaitForHumanEvent] = []

    async def _checkpoint(*, event: WaitForHumanEvent, **_kw: object) -> None:
        checkpointed.append(event)

    monkeypatch.setattr("openexecutive.workflows.gate.checkpoint_gate", _checkpoint)
    _resolved(resume_state=_scheduled_resume_state())
    _install_stub(monkeypatch, _real_workflow([later]))
    asyncio.run(_process_resumable(datetime.now(UTC)))
    assert checkpointed[0].resume_state is not None
    assert checkpointed[0].resume_state.deliver_to_person_id == 11
    assert sent == []


@pytest.mark.parametrize(("owner", "expected_person"), [(5, 5), (None, None)])
async def test_a_resumed_run_reads_as_its_owner(
    monkeypatch: pytest.MonkeyPatch, owner: int | None, expected_person: int | None
) -> None:
    """A resumed run's steps read and draft as whoever started it, and as
    nobody for a team run — never as the principal by default."""
    from openexecutive.orchestrator.artifact_records import Viewer, current_viewer
    from openexecutive.workflows import resumer

    seen: list[Viewer] = []

    async def _drive(row: dict, claim: str, db_path: object = None) -> bool:
        seen.append(current_viewer())
        return True

    monkeypatch.setattr(resumer, "_drive_resume", _drive)
    monkeypatch.setattr(
        resumer._wf_persistence, "get_run",
        lambda run_id, db_path=None: {"run_id": run_id, "owner_person_id": owner},
    )
    assert await resumer._execute_resume({"run_id": "r"}, "claim") is True
    (viewer,) = seen
    assert viewer.person_id == expected_person and not viewer.is_principal