Skip to content

HarnessRouter Community Edition: the self-hosted, Apache-2.0 edition of the unified interface for agent harnesses. Run Codex, Claude Code, Hermes, PI, DSH, and more through one API, with sessions, streaming, files, cancellation, and failure handling. Implements the Unified Harness Protocol (UHP), an open standard. Your keys, your infrastructure.

18.8 KB
Raw Blame
"""An incomplete turn says WHY, and only when it knows.

One banner for every incomplete — "hit its step or time limit" — misdiagnosed a turn that was
cut by a deploy restart, and sent the operator debugging a 400-step default that was never
reached. The record now carries incomplete_details.reason (max_steps | timeout | interrupted);
absent means unknown, and the console claims nothing specific.
"""
import os
import pathlib
import sys

sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
os.environ.setdefault("HR_BACKING", "local")
import app as A  # noqa: E402


def _tr(**kw):
    return A._RespTranslator("resp_t", "m", None, True, 0.0, **kw)


def test_a_capped_turn_names_its_cap():
    tr = _tr()
    tr.incomplete_reason = "max_steps"
    d = tr._response_obj("incomplete")
    assert d["incomplete_details"] == {"reason": "max_steps", "handoff": None}   # a capped run hands nothing off


def test_an_unknown_cause_stays_null_rather_than_guessing():
    d = _tr()._response_obj("incomplete")
    assert d["incomplete_details"] is None


def test_a_completed_turn_never_carries_one():
    tr = _tr()
    tr.incomplete_reason = "max_steps"          # stale attr must not leak onto a clean finish
    assert _tr()._response_obj("completed")["incomplete_details"] is None
    assert tr._response_obj("completed")["incomplete_details"] is None


def test_the_empty_output_settle_never_claims_a_cause():
    """The reconciler's empty-output settle cannot tell a cut turn from one that ended cleanly
    with no text: a max_step=1 turn ends 'done' with empty output and nothing interrupted it —
    measured live on the SaaS port, which shipped "interrupted" there and caught it within the
    hour. Unknown must stay absent rather than be guessed."""
    src = open(pathlib.Path(__file__).resolve().parents[1] / "app.py").read()
    i = src.index('settled == "completed" and not (rec.get("output")')
    window = src[i:i + 1200]
    assert '"reason"' not in window.split("else:")[0], (
        "the empty-output settle claims a reason it cannot know")


def test_a_finishing_owner_is_not_settled_from_under(monkeypatch):
    """The session flips to done before its owner has written the checkpoint and the full
    response record; a poller's reconcile in that window used to settle the record 'incomplete'
    and a replica cached that for good. A fresh heartbeat means the owner is alive: leave it."""
    import asyncio, time
    calls = []
    async def _vertex_get(vid):
        return {"id": vid, "status": "done", "heartbeat": str(time.time() - 5)}
    async def _blob_put(*a, **k):
        calls.append("put"); return True
    monkeypatch.setattr(A, "_vertex_get", _vertex_get)
    monkeypatch.setattr(A, "_blob_put", _blob_put)
    rec = {"id": "resp_x", "status": "running", "_session_id": "hsess1", "output": []}
    out = asyncio.run(A._reconcile_response("resp_x", rec))
    assert out["status"] == "running" and calls == []
    # the same session with a dead owner is settled, as before
    async def _vertex_get_dead(vid):
        return {"id": vid, "status": "done", "heartbeat": str(time.time() - 10_000)}
    async def _blob_get(*a, **k):
        return None
    async def _vg_upsert(*a, **k):
        return None
    monkeypatch.setattr(A, "_vertex_get", _vertex_get_dead)
    monkeypatch.setattr(A, "_blob_get", _blob_get)
    monkeypatch.setattr(A, "_vg_upsert", _vg_upsert)
    out = asyncio.run(A._reconcile_response("resp_x", dict(rec)))
    assert out["status"] == "incomplete" and calls == ["put"]


def test_a_refused_turn_leaves_the_session_as_it_found_it():
    """The lease is checked before the session is stamped running/starting: a follow-up refused
    as 'already in progress' must not leave a finished session looking live, or the orphan sweep
    adopts it and every retry is refused in turn (omp follow-ups, 2026-09-15)."""
    import inspect
    src = inspect.getsource(A._resp_execute)
    admit = src.index("control_store.lease_admit(")
    stamp = src.index('"turn_status": "starting"')
    assert admit < stamp, "the session must be stamped only after admission"
    refused = src.index('"a turn is already in progress for this session"')
    assert refused < stamp, "a refused turn returns before the stamp"


class _Resp:
    def __init__(self, code):
        self.status_code = code


def _sweep(monkeypatch, v, probe_code=404, rec=None, tts=None):
    """Run the orphan sweep on a session vertex, the runner answering the turn probe with
    probe_code; returns the vertex writes and the response record as it was persisted."""
    import asyncio
    vertex = dict(v)
    writes: list = []
    stored: dict = {}
    async def _vup(sid, props):
        writes.append(props)
        vertex.update(props)
    async def _vget(sid):
        return dict(vertex)
    async def _tts(blob, since_ms=0):
        return None
    async def _sandbox(path, sid, method, **k):
        return _Resp(probe_code)
    async def _resp_get(rid):
        return dict(rec) if rec else None
    async def _blob_get(*a, **k):
        return None
    async def _blob_put(key, data, **k):
        import json
        stored.update(json.loads(data))
        return True
    async def _vg_upsert(*a, **k):
        return None
    for name, fn in (("_vertex_upsert", _vup), ("_vertex_get", _vget), ("_trace_terminal_status", tts or _tts),
                     ("_sandbox", _sandbox), ("_resp_get", _resp_get), ("_blob_get", _blob_get),
                     ("_blob_put", _blob_put), ("_vg_upsert", _vg_upsert)):
        monkeypatch.setattr(A, name, fn)
    settled = asyncio.run(A._reconcile_session(v))
    return settled, writes, stored


def test_a_turn_the_runner_lost_to_a_restart_fails_saying_it_was_interrupted(monkeypatch):
    """#505: the runner restarted under the turn, so it answers 404 for it. Within UHP 2026-10-19 the
    turn is `failed` (lifecycle §3 keeps `incomplete` for budgets), and the error says a restart cut
    it: harness_error (the type of an error inside a failed response, and the §3.3 code), with
    detail.reason interrupted, in the same words as the hosted service."""
    import time
    v = {"id": "hsess1", "status": "running", "turn_status": "running", "runner_turn_id": "rt_1",
         "running_response_id": "resp_1", "heartbeat": str(time.time() - 600)}
    rec = {"id": "resp_1", "status": "running", "_session_id": "hsess1", "output": []}
    settled, writes, stored = _sweep(monkeypatch, v, 404, rec)
    assert settled is True
    assert writes[0] == {"status": "failed", "turn_status": "failed", "end_reason": "interrupted"}
    assert stored["status"] == "failed" and not stored.get("incomplete_details")
    assert stored["error"] == {"type": "harness_error", "code": "harness_error",
                               "message": A._INTERRUPTED_MESSAGE, "detail": {"reason": "interrupted"}}


def test_a_turn_with_no_runner_turn_on_record_fails_saying_it_was_interrupted(monkeypatch):
    """No runner turn was ever on record: the turn never reached the runner, and the process that
    owned it is gone. It is failed, and a failed task's error says why (lifecycle §3): interrupted,
    as the hosted service names the same case."""
    import time
    v = {"id": "hsess1", "status": "running", "turn_status": "starting", "runner_turn_id": "",
         "heartbeat": str(time.time() - 600)}
    settled, writes, _ = _sweep(monkeypatch, v, 404)
    assert settled is True and writes[0] == {"status": "failed", "turn_status": "failed", "end_reason": "interrupted"}


def test_a_turn_past_the_hard_cap_fails_saying_it_ran_out_of_time(monkeypatch):
    """Past the hard cap with no result: failed, never bare, with detail.reason timeout."""
    import time
    v = {"id": "hsess1", "status": "running", "turn_status": "running", "runner_turn_id": "rt_1",
         "running_response_id": "resp_1", "heartbeat": str(time.time() - A._sweep_cap_s() - 60),
         "turn_at": str(time.time() - A._sweep_cap_s() - 60)}
    rec = {"id": "resp_1", "status": "running", "_session_id": "hsess1", "output": [],
           "created_at": int(time.time() - A._sweep_cap_s() - 60)}
    settled, writes, stored = _sweep(monkeypatch, v, 200, rec)
    assert settled is True and writes[0] == {"status": "failed", "turn_status": "failed", "end_reason": "timeout"}
    assert stored["error"] == {"type": "harness_error", "code": "harness_error",
                               "message": A._CAP_MESSAGE, "detail": {"reason": "timeout"}}


def test_a_runner_that_still_holds_the_turn_is_not_settled(monkeypatch):
    import time
    monkeypatch.setattr(A, "_adopting", {"hsess1"})        # adoption already running: spawn nothing
    v = {"id": "hsess1", "status": "running", "turn_status": "running", "runner_turn_id": "rt_1",
         "heartbeat": str(time.time() - 600)}
    settled, writes, _ = _sweep(monkeypatch, v, 200)
    assert settled is False and writes == []


def test_a_new_turn_drops_the_last_turns_reason():
    """The reason lives on the session, so the next turn's start clears it: a later orphan must
    not inherit 'interrupted' from an earlier one."""
    import inspect
    src = inspect.getsource(A)
    i = src.index('"heartbeat": str(time.time()), "runner_turn_id": "", "end_reason": ""')
    assert '"turn_status": "starting"' in src[i - 200:i]
    j = src.index('"running_response_id": resp_id, "runner_turn_id": "",')
    assert '"end_reason": ""' in src[j:j + 250]


def test_a_turn_the_gateway_drains_under_fails_saying_it_was_interrupted(monkeypatch):
    import asyncio
    tr = _tr()
    emitted, persisted = [], []
    async def _emit(ev):
        emitted.append(ev)
    async def _persist(status):
        persisted.append((status, tr._response_obj(status)))
    asyncio.run(A._settle_drained(_persist, tr, _emit))
    assert emitted[-1]["type"] == "response.failed"
    assert [e for e in emitted if e["type"] == "error"][0]["code"] == "harness_error"
    assert persisted[0][0] == "failed"
    assert persisted[0][1]["incomplete_details"] is None
    assert persisted[0][1]["error"]["detail"] == {"reason": "interrupted"}
    assert persisted[0][1]["error"]["type"] == "harness_error"


def test_a_dead_follow_up_is_not_settled_by_the_turn_before_it(monkeypatch):
    """The trace is one stream for the whole session. Turn 1 ended with a result; turn 2 died
    under a restart with none. Read unscoped, turn 1's result settled turn 2 'completed' with
    no output (#505, live). Scoped to turn 2's start it finds nothing, and the probe decides."""
    import asyncio, json
    chunks = {"t/events/1000-a-0-1.jsonl": json.dumps({"type": "result", "is_error": False}).encode(),
              "t/events/5000-b-0-1.jsonl": json.dumps({"type": "tool_use", "name": "Bash"}).encode()}
    async def _list(prefix, **k):
        return [{"file_id": c} for c in chunks]
    async def _get(cid, **k):
        return chunks.get(cid)
    monkeypatch.setattr(A, "_blob_list_all", _list)
    monkeypatch.setattr(A, "_blob_get", _get)
    assert asyncio.run(A._trace_terminal_status("t")) == "done"            # unscoped: turn 1's
    assert asyncio.run(A._trace_terminal_status("t", 4000)) is None         # turn 2's own: none
    assert A._turn_since_ms(None) == 0 and A._turn_since_ms({"created_at": 4.5}) == 4500


def test_the_previous_turns_result_in_the_same_second_is_not_this_turns():
    """created_at is whole seconds. Measured: turn 1's result chunk at 1792417802072, turn 2
    created later in second 1792417802. The cutoff is the next second, so that chunk is not read."""
    since = A._turn_since_ms({"created_at": 1792417802})
    assert since > 1792417802072 and since == 1792417803000


def test_the_sweep_reads_the_trace_from_the_running_turns_start(monkeypatch):
    import time
    seen = []
    v = {"id": "hsess1", "status": "running", "turn_status": "running", "runner_turn_id": "rt_1",
         "running_response_id": "resp_2", "trace_blob": "org/inv_hsess1", "heartbeat": str(time.time() - 600)}
    rec = {"id": "resp_2", "status": "running", "_session_id": "hsess1", "output": [], "created_at": 4}
    async def _tts(blob, since_ms=0):
        seen.append(since_ms)
        return None
    settled, writes, _ = _sweep(monkeypatch, v, 404, rec, tts=_tts)
    assert seen[0] == 5000 and writes[0]["end_reason"] == "interrupted"


def test_the_record_carries_its_start_to_the_millisecond(monkeypatch):
    """created_at is whole seconds; _created_ms is the exact start, and the cutoff when present.
    Both the result walk and the output replay then read exactly this turn's chunks."""
    import asyncio, json
    put = {}
    async def _blob_put(key, data, **k):
        put.update(json.loads(data)); return True
    async def _vg_upsert(*a, **k):
        return None
    monkeypatch.setattr(A, "_blob_put", _blob_put)
    monkeypatch.setattr(A, "_vg_upsert", _vg_upsert)
    asyncio.run(A._resp_put("resp_1", {"id": "resp_1", "created_at": 1792417802}, "o", "s", None,
                            "running", 1792417802.5, True))
    assert put["_created_ms"] == 1792417802500
    rec = {"created_at": 1792417802, "_created_ms": 1792417802500}
    assert A._turn_since_ms(rec) == A._turn_since_ms(rec, for_result=False) == 1792417802500
    old = {"created_at": 1792417802}                      # a record from before the field
    assert A._turn_since_ms(old) == 1792417803000          # the result walk skips the shared second
    assert A._turn_since_ms(old, for_result=False) == 1792417802000   # the replay keeps it, as before


def test_the_reason_is_written_only_onto_the_turn_it_was_proved_for(monkeypatch):
    """The session's end reason belongs to its running response. An older orphan of the same
    session settled later must not be told it was interrupted."""
    import asyncio, time
    v = {"id": "hsess1", "status": "failed", "end_reason": "interrupted",
         "running_response_id": "resp_2", "heartbeat": str(time.time() - 600)}
    async def _vget(sid):
        return dict(v)
    async def _blob_get(*a, **k):
        return None
    async def _blob_put(*a, **k):
        return True
    async def _vg_upsert(*a, **k):
        return None
    for name, fn in (("_vertex_get", _vget), ("_blob_get", _blob_get), ("_blob_put", _blob_put),
                     ("_vg_upsert", _vg_upsert)):
        monkeypatch.setattr(A, name, fn)
    old = asyncio.run(A._reconcile_response("resp_1", {"id": "resp_1", "status": "running",
                                                       "_session_id": "hsess1", "output": []}))
    assert old["status"] == "failed" and ((old.get("error") or {}).get("detail") or {}).get("reason") != "interrupted"
    cur = asyncio.run(A._reconcile_response("resp_2", {"id": "resp_2", "status": "running",
                                                       "_session_id": "hsess1", "output": []}))
    assert cur["status"] == "failed" and cur["error"]["detail"] == {"reason": "interrupted"}


def test_a_restart_never_reports_incomplete(monkeypatch):
    """`incomplete` is a budget stop (lifecycle §3, UHP 2026-10-19). Neither restart path produces
    it: the sweep's 404 and the shutdown drain both end `failed`."""
    import asyncio, time
    v = {"id": "hsess1", "status": "running", "turn_status": "running", "runner_turn_id": "rt_1",
         "running_response_id": "resp_1", "heartbeat": str(time.time() - 600)}
    rec = {"id": "resp_1", "status": "running", "_session_id": "hsess1", "output": []}
    _, writes, stored = _sweep(monkeypatch, v, 404, rec)
    assert "incomplete" not in {writes[0]["status"], writes[0]["turn_status"], stored["status"]}
    tr = _tr()
    seen = []
    async def _emit(ev):
        seen.append(ev["type"])
    async def _persist(status):
        seen.append(status)
    asyncio.run(A._settle_drained(_persist, tr, _emit))
    assert "incomplete" not in seen and "response.incomplete" not in seen


def test_a_drain_after_the_turn_ended_keeps_its_own_end():
    """A drain that cuts in after _resp_execute returned must not overwrite a finished turn as
    interrupted. If it cut the closing events short, the terminal event is still sent, so an open
    Workbench does not stay on 'running'; if they were all sent, nothing is sent twice."""
    import asyncio, inspect
    for told, sent in ((False, ["response.completed"]), (True, [])):
        tr = _tr()
        emitted, persisted = [], []
        async def _emit(ev):
            emitted.append(ev["type"])
        async def _persist(status):
            persisted.append(status)
        asyncio.run(A._settle_drained(_persist, tr, _emit, {"status": "completed", "told": told}))
        assert emitted == sent and persisted == ["completed"] and tr.incomplete_reason == ""
    src = inspect.getsource(A.create_response)
    assert src.count("await _settle_drained(persist, tr, ") == 2
    assert src.count(", end)") == 2, "both turn paths hand the drain the turn's own end"


def test_a_running_turn_with_no_readable_record_is_scoped_by_its_turn_at(monkeypatch):
    """No record to read the start from: the session's turn_at (stamped at accept) scopes the
    walk, which unscoped would find the previous turn's result."""
    import time
    seen = []
    async def _tts(blob, since_ms=0):
        seen.append(since_ms)
        return None
    turn_at = round(time.time() - 600, 3)               # inside the hard cap, which runs from turn_at
    v = {"id": "hsess1", "status": "running", "turn_status": "running", "runner_turn_id": "rt_1",
         "running_response_id": "", "trace_blob": "org/inv_hsess1", "turn_at": str(turn_at),
         "heartbeat": str(time.time() - 600)}
    settled, writes, _ = _sweep(monkeypatch, v, 404, None, tts=_tts)
    assert seen[0] == int(turn_at * 1000) and writes[0]["end_reason"] == "interrupted"


def test_the_reconcilers_own_hard_cap_is_never_a_bare_failed(monkeypatch):
    """A background poll's reconcile that settles a record past the hard cap gives it the timeout
    error, so a failed task always says why (lifecycle §3)."""
    import asyncio, time
    long_ago = time.time() - A._sweep_cap_s() - 60
    v = {"id": "hsess1", "status": "running", "turn_status": "running", "heartbeat": str(long_ago),
         "created_at": str(long_ago), "running_response_id": "resp_1"}
    async def _vget(sid):
        return dict(v)
    async def _tts(blob, since_ms=0):
        return None
    async def _blob_get(*a, **k):
        return None
    async def _blob_put(*a, **k):
        return True
    async def _vg_upsert(*a, **k):
        return None
    for name, fn in (("_vertex_get", _vget), ("_trace_terminal_status", _tts), ("_blob_get", _blob_get),
                     ("_blob_put", _blob_put), ("_vg_upsert", _vg_upsert)):
        monkeypatch.setattr(A, name, fn)
    out = asyncio.run(A._reconcile_response("resp_1", {"id": "resp_1", "status": "running", "_session_id": "hsess1",
                                                       "output": [], "created_at": int(long_ago)}))
    assert out["status"] == "failed"
    assert out["error"] == {"type": "harness_error", "code": "harness_error", "message": A._CAP_MESSAGE,
                            "detail": {"reason": "timeout"}}