Skip to content

Self-hosted AI workspace.

Python
12.7 KB
Raw Blame
"""Agent execution, transcripts and same-task stream cleanup."""

import logging
from contextlib import aclosing
from typing import Optional

from .dependencies import AuditDependencies

logger = logging.getLogger(__name__)

_skill_test_jobs: dict = {}


class SkillAuditUnavailable(RuntimeError):
    """The test infrastructure failed; this is not evidence about a skill."""


async def run_test_job(
    key,
    name,
    md,
    task,
    url,
    model,
    headers,
    owner,
    skills_manager=None,
    *,
    messages=None,
    transcript=None,
    exact_approval=None,
    request_authority=None,
    deps: AuditDependencies,
):
    """Background coroutine: run the skill in an agent loop, capture a condensed
    log + transcript, then have the judge grade it. Writes into deps.test_jobs."""
    import json as _json
    from src.agent_loop import stream_agent_loop
    from src.agent_runtime.authority import RequestAuthority

    job = deps.test_jobs.get(key)
    if job is None:
        return
    log = job["log"]
    transcript = transcript if isinstance(transcript, list) else []
    say_buf = []
    skill_stats = {"turns": 0, "tool_calls": 0}

    def _flush_say():
        if say_buf:
            log.append({"type": "say", "text": "".join(say_buf)})
            say_buf.clear()

    messages = list(messages) if isinstance(messages, list) else deps.test_messages(md, task)
    try:
        # aclosing: pausing for an approval leaves this loop early, and the
        # stream's ContextVar cleanup must run in this task, not in asyncio's
        # finalizer (#6635).
        async with aclosing(stream_agent_loop(
            url, model, messages, headers=headers,
            temperature=0.3, max_tokens=0, max_rounds=8, owner=owner,
            exact_approval=exact_approval,
            request_authority=(request_authority or (
                exact_approval.pending.request_authority if exact_approval is not None else None
            ) or RequestAuthority.empty(owner=owner)),
        )) as stream:
            async for chunk in stream:
                if not chunk.startswith("data: ") or chunk.strip() == "data: [DONE]":
                    continue
                try:
                    d = _json.loads(chunk[6:])
                except Exception:
                    continue
                if d.get("delta"):
                    say_buf.append(d["delta"]); transcript.append(d["delta"])
                elif d.get("type") == "tool_start":
                    _flush_say()
                    skill_stats["tool_calls"] += 1
                    cmd = str(d.get("command") or d.get("args") or "")[:300]
                    log.append({"type": "tool_start", "tool": d.get("tool"), "command": cmd})
                    transcript.append(f"\n[tool {d.get('tool')}] {cmd}\n")
                elif d.get("type") == "tool_output":
                    _flush_say()
                    out = str(d.get("output") or "")[:600]
                    tool_log = {"type": "tool_output", "output": out}
                    approval = d.get("ask_user")
                    if isinstance(approval, dict):
                        tool_log["ask_user"] = approval
                    log.append(tool_log)
                    transcript.append(f"[output] {out}\n")
                    if (
                        isinstance(approval, dict)
                        and approval.get("kind") == "tool_approval"
                        and approval.get("approval_id")
                    ):
                        # Manual skill tests have their own polling UI instead of a
                        # chat session. Pause the run and retain only server-side
                        # continuation state until the same owner approves/denies
                        # this exact sealed action.
                        job["status"] = "awaiting_approval"
                        job["approval"] = approval
                        job["_transcript"] = transcript
                        return
                elif d.get("type") == "agent_step":
                    _flush_say()
                    try:
                        skill_stats["turns"] = max(skill_stats["turns"], int(d.get("round") or 0))
                    except (TypeError, ValueError):
                        pass
                    log.append({"type": "agent_step", "round": d.get("round")})
                    transcript.append(f"\n--- round {d.get('round')} ---\n")
                elif d.get("type") == "metrics":
                    data = d.get("data") or {}
                    try:
                        skill_stats["turns"] = max(skill_stats["turns"], int(data.get("agent_rounds") or 0))
                    except (TypeError, ValueError):
                        pass
                    try:
                        skill_stats["tool_calls"] = max(skill_stats["tool_calls"], int(data.get("tool_calls") or 0))
                    except (TypeError, ValueError):
                        pass
                if len(log) > 600:
                    del log[0:len(log) - 600]
        _flush_say()
    except Exception as e:
        _flush_say()
        log.append({"type": "error", "error": str(e)})

    job.pop("approval", None)
    job.pop("_transcript", None)
    job.pop("_run", None)
    log.append({"type": "evaluating"})
    try:
        baseline_task = task
        log.append({"type": "agent_step", "round": "baseline"})
        baseline_transcript, baseline_stats, baseline_approval = await deps.run_arm(
            deps.baseline_messages(baseline_task),
            url,
            model,
            headers,
            owner,
        )
        if baseline_approval is not None:
            try:
                from src.tool_approvals import tool_approval_store
                tool_approval_store.consume(
                    baseline_approval.get("approval_id"),
                    decision="deny",
                    owner=owner,
                    session_id=None,
                )
            except Exception:
                logger.debug("Could not retire manual-test baseline approval", exc_info=True)
        job["verdict"] = await deps.evaluate_run(
            md,
            task,
            "".join(transcript),
            url,
            model,
            headers,
            baseline_transcript=baseline_transcript,
            skill_stats=skill_stats,
            baseline_stats=baseline_stats,
            owner=owner,
        )
    except Exception as e:
        job["verdict"] = {"verdict": "unknown", "confidence": 0, "summary": f"Eval failed: {e}", "issues": []}
    # Record the result so the card shows a 'verified' check (a manual test
    # never involves the teacher) and nudge the confidence score to match the
    # verdict — same scale as Audit-all's pass=0.95. inconclusive/unknown leave
    # the score alone (missing-fixture or parse failures shouldn't punish it).
    if skills_manager is not None:
        v = (job["verdict"] or {}).get("verdict") or "unknown"
        try:
            skills_manager.set_audit(
                name,
                v,
                by_teacher=False,
                worker_model=model,
                owner=owner,
                **deps.verdict_efficiency(job.get("verdict")),
            )
        except Exception:
            pass
        conf = {"pass": 0.95, "needs_work": 0.6, "fail": 0.4}.get(v)
        if conf is not None:
            try:
                skills_manager.update_skill(name, {"confidence": conf}, owner=owner)
            except Exception:
                pass
    job["status"] = "done"


async def run_audit_arm(messages: list[dict], url, model, headers, owner,
                               workload: str = "foreground") -> tuple[str, dict, Optional[dict]]:
    """Run one audit arm in the agent loop; return transcript, stats, approval."""
    import json as _json
    from src.agent_loop import stream_agent_loop
    from src.agent_runtime.authority import RequestAuthority, active_request_authority
    transcript = []
    approval_required = None
    stats = {"turns": 0, "tool_calls": 0}
    try:
        # max_tokens explicitly set: passing 0 lets some upstreams (Ollama,
        # OpenAI-compat) generate an empty completion, which manifested as
        # the skill test returning nothing while chat (which carries its
        # preset's max_tokens) worked. 4096 matches the chat default.
        # aclosing: stopping for an approval leaves this loop early (#6635).
        async with aclosing(stream_agent_loop(
            url, model, messages, headers=headers,
            temperature=0.3, max_tokens=4096, max_rounds=8,
            owner=owner, workload=workload, suppress_skills=True,
            request_authority=(active_request_authority() or RequestAuthority.empty(owner=owner)),
        )) as stream:
            async for chunk in stream:
                # Streams can include an SSE event line before the data line,
                # notably `event: error`. Do not silently discard those failures.
                payload = next((line[6:] for line in chunk.splitlines() if line.startswith("data: ")), None)
                if payload is None or payload == "[DONE]":
                    continue
                try:
                    d = _json.loads(payload)
                except Exception:
                    continue
                if d.get("error") or d.get("type") == "error":
                    raise SkillAuditUnavailable(str(d.get("error") or d.get("message") or "Audit stream failed"))
                if d.get("delta"):
                    transcript.append(d["delta"])
                elif d.get("type") == "tool_start":
                    stats["tool_calls"] += 1
                    transcript.append(f"\n[tool {d.get('tool')}] {str(d.get('command') or d.get('args') or '')[:300]}\n")
                elif d.get("type") == "tool_output":
                    transcript.append(f"[output] {str(d.get('output') or '')[:600]}\n")
                    approval = d.get("ask_user")
                    if (
                        isinstance(approval, dict)
                        and approval.get("kind") == "tool_approval"
                    ):
                        approval_required = approval
                        break
                elif d.get("type") == "agent_step":
                    try:
                        stats["turns"] = max(stats["turns"], int(d.get("round") or 0))
                    except (TypeError, ValueError):
                        pass
                    transcript.append(f"\n--- round {d.get('round')} ---\n")
                elif d.get("type") == "metrics":
                    data = d.get("data") or {}
                    try:
                        stats["turns"] = max(stats["turns"], int(data.get("agent_rounds") or 0))
                    except (TypeError, ValueError):
                        pass
                    try:
                        stats["tool_calls"] = max(stats["tool_calls"], int(data.get("tool_calls") or 0))
                    except (TypeError, ValueError):
                        pass
    except SkillAuditUnavailable:
        raise
    except Exception as e:
        raise SkillAuditUnavailable(str(e)) from e
    return "".join(transcript), stats, approval_required


async def run_test_once(md: str, task: str, url, model, headers, owner,
                               workload: str = "foreground", *, deps: AuditDependencies) -> tuple:
    """Run the skill once in the agent loop; return (transcript, verdict)."""
    messages = deps.test_messages(md, task)
    text, stats, approval_required = await deps.run_arm(
        messages, url, model, headers, owner, workload=workload,
    )
    if approval_required is not None:
        # Unattended audits have no authority to approve and no UI that could
        # resume this record. Destructively deny it now instead of leaving a
        # reusable opaque grant pending until TTL/cap eviction.
        try:
            from src.tool_approvals import tool_approval_store
            tool_approval_store.consume(
                approval_required.get("approval_id"),
                decision="deny",
                owner=owner,
                session_id=None,
            )
        except Exception:
            logger.debug("Could not retire unattended skill approval", exc_info=True)
        return text, {
            "verdict": "inconclusive",
            "confidence": 1.0,
            "summary": (
                "This automated audit reached an exact action that requires "
                "a human approval; no action was executed."
            ),
            "issues": [
                "Run this skill's manual test and review the sealed action."
            ],
            "approval_required": True,
        }
    verdict = await deps.evaluate_run(
        md,
        task,
        text,
        url,
        model,
        headers,
        skill_stats=stats,
        workload=workload,
        owner=owner,
    )
    return text, verdict