Skip to content

Automate browser based workflows with AI

feat(SKY-12215): self-heal reliability aggregation service (runs × episodes) (#7415)

pedrohsdbedfe565fa3a5
diff --git a/skyvern/forge/sdk/db/repositories/self_heal.py b/skyvern/forge/sdk/db/repositories/self_heal.py
index aefe1ffed..67bae738e 100644
--- a/skyvern/forge/sdk/db/repositories/self_heal.py
+++ b/skyvern/forge/sdk/db/repositories/self_heal.py
@@ -7,7 +7,7 @@ from sqlalchemy import select, update
from skyvern.forge.sdk.db._error_handling import db_operation
from skyvern.forge.sdk.db.base_repository import BaseRepository
from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
-from skyvern.forge.sdk.db.models import HealEpisodeModel, WorkflowHealProposalModel
+from skyvern.forge.sdk.db.models import HealEpisodeModel, WorkflowHealProposalModel, WorkflowRunModel
from skyvern.forge.sdk.utils.sanitization import sanitize_postgres_text
from skyvern.schemas.self_heal import HealEpisode, HealSkipReason, HealStatus, OutputObligation, WorkflowHealProposal
@@ -143,6 +143,43 @@ class SelfHealRepository(BaseRepository):
).all()
return [HealEpisode.model_validate(episode) for episode in episodes]
+ @db_operation("get_recent_terminal_workflow_run_ids")
+ async def get_recent_terminal_workflow_run_ids(
+ self,
+ organization_id: str,
+ workflow_permanent_id: str,
+ limit: int = 20,
+ ) -> list[str]:
+ terminal_statuses = ("completed", "failed", "terminated", "timed_out")
+ bounded_limit = max(1, min(limit, 100))
+ async with self.Session() as session:
+ query = (
+ select(WorkflowRunModel.workflow_run_id)
+ .where(
+ WorkflowRunModel.organization_id == organization_id,
+ WorkflowRunModel.workflow_permanent_id == workflow_permanent_id,
+ WorkflowRunModel.status.in_(terminal_statuses),
+ WorkflowRunModel.parent_workflow_run_id.is_(None),
+ WorkflowRunModel.copilot_session_id.is_(None),
+ WorkflowRunModel.debug_session_id.is_(None),
+ )
+ .order_by(WorkflowRunModel.created_at.desc(), WorkflowRunModel.workflow_run_id.desc())
+ .limit(bounded_limit)
+ )
+ return list((await session.scalars(query)).all())
+
+ @db_operation("get_heal_episodes_for_runs")
+ async def get_heal_episodes_for_runs(self, organization_id: str, workflow_run_ids: list[str]) -> list[HealEpisode]:
+ if not workflow_run_ids:
+ return []
+ async with self.Session() as session:
+ query = select(HealEpisodeModel).where(
+ HealEpisodeModel.organization_id == organization_id,
+ HealEpisodeModel.workflow_run_id.in_(workflow_run_ids),
+ )
+ episodes = (await session.scalars(query.order_by(HealEpisodeModel.created_at.asc()))).all()
+ return [HealEpisode.model_validate(episode) for episode in episodes]
+
@db_operation("create_heal_proposal")
async def create_heal_proposal(
self,
diff --git a/skyvern/forge/sdk/routes/self_heal.py b/skyvern/forge/sdk/routes/self_heal.py
index ec52e23ac..d4715f543 100644
--- a/skyvern/forge/sdk/routes/self_heal.py
+++ b/skyvern/forge/sdk/routes/self_heal.py
@@ -6,7 +6,14 @@ from skyvern.forge import app
from skyvern.forge.sdk.routes.routers import base_router
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.services import org_auth_service
-from skyvern.schemas.self_heal import HealEpisodeView, HealStatus, RunHealSummary, summarize_run_heals
+from skyvern.schemas.self_heal import (
+ HealEpisodeView,
+ HealStatus,
+ RunHealSummary,
+ WorkflowReliability,
+ summarize_run_heals,
+)
+from skyvern.services.self_heal_reliability_service import get_workflow_reliability
LOG = structlog.get_logger()
@@ -57,3 +64,15 @@ async def get_run_heal_episodes(
episodes=[HealEpisodeView.from_episode(episode) for episode in episodes],
summary=summarize_run_heals(episodes),
)
+
+
+@base_router.get(
+ "/workflows/{workflow_permanent_id}/reliability",
+ response_model=WorkflowReliability,
+ include_in_schema=False,
+)
+async def get_workflow_reliability_route(
+ workflow_permanent_id: str,
+ organization: Organization = Depends(org_auth_service.get_current_org),
+) -> WorkflowReliability:
+ return await get_workflow_reliability(organization.organization_id, workflow_permanent_id)
diff --git a/skyvern/schemas/self_heal.py b/skyvern/schemas/self_heal.py
index e5b22c216..da81758a5 100644
--- a/skyvern/schemas/self_heal.py
+++ b/skyvern/schemas/self_heal.py
@@ -33,6 +33,16 @@ class OutputObligation(StrEnum):
observed = "observed"
+class ReliabilityState(StrEnum):
+ healthy = "healthy"
+ watch = "watch"
+ action_needed = "action_needed"
+
+
+RELIABILITY_WINDOW = 20
+RELIABILITY_MIN_RUNS = 10
+
+
@dataclass(frozen=True)
class HealClassification:
healable: bool
@@ -157,6 +167,23 @@ class RunHealSummary(BaseModel):
blocks_with_heal_attempt: int
+class RunHealGroup(BaseModel):
+ workflow_run_id: str
+ episodes: list[HealEpisode]
+
+
+class WorkflowReliability(BaseModel):
+ state: ReliabilityState
+ outcome_risk: bool
+ scored: bool
+ window_runs: int
+ healed_runs: int
+ heal_rate: float
+ consecutive_healed_runs: int
+ floor_runs: int
+ outcome_risk_runs: int
+
+
def summarize_run_heals(episodes: list[HealEpisode]) -> RunHealSummary:
episodes_by_block: dict[str, list[HealEpisode]] = {}
for episode in episodes:
@@ -184,6 +211,59 @@ def summarize_run_heals(episodes: list[HealEpisode]) -> RunHealSummary:
)
+def compute_workflow_reliability(runs: list[RunHealGroup]) -> WorkflowReliability:
+ run_flags: list[tuple[bool, bool, bool]] = []
+ for run in runs:
+ healed = any(
+ episode.status in {HealStatus.fired_completed, HealStatus.fired_unverified, HealStatus.fired_failed}
+ for episode in run.episodes
+ )
+ floor = any(episode.engine == "floor" for episode in run.episodes)
+ outcome_risk = len(summarize_run_heals(run.episodes).blocks_outcome_risk) > 0
+ run_flags.append((healed, floor, outcome_risk))
+
+ window_runs = len(runs)
+ healed_runs = sum(healed for healed, _, _ in run_flags)
+ heal_rate = healed_runs / window_runs if window_runs else 0.0
+
+ consecutive_healed_runs = 0
+ for healed, _, _ in run_flags:
+ if not healed:
+ break
+ consecutive_healed_runs += 1
+
+ floor_runs = sum(floor for _, floor, _ in run_flags)
+ recent_run_flags = run_flags[:10]
+ recent_healed_runs = sum(healed for healed, _, _ in recent_run_flags)
+ outcome_risk_runs = sum(outcome_risk for _, _, outcome_risk in recent_run_flags)
+
+ scored = window_runs >= RELIABILITY_MIN_RUNS
+ if not scored:
+ state = ReliabilityState.healthy
+ elif recent_healed_runs >= 3 or heal_rate >= 0.20 or floor_runs >= 2:
+ state = ReliabilityState.action_needed
+ elif healed_runs >= 2 or consecutive_healed_runs >= 2:
+ state = ReliabilityState.watch
+ else:
+ state = ReliabilityState.healthy
+
+ return WorkflowReliability(
+ state=state,
+ outcome_risk=outcome_risk_runs > 0,
+ scored=scored,
+ window_runs=window_runs,
+ healed_runs=healed_runs,
+ heal_rate=heal_rate,
+ consecutive_healed_runs=consecutive_healed_runs,
+ floor_runs=floor_runs,
+ outcome_risk_runs=outcome_risk_runs,
+ )
+
+
+def reliability_state_transition(previous: ReliabilityState | None, current: ReliabilityState) -> bool:
+ return previous is None or previous != current
+
+
class WorkflowHealProposal(BaseModel):
model_config = ConfigDict(from_attributes=True)
diff --git a/skyvern/services/self_heal_reliability_service.py b/skyvern/services/self_heal_reliability_service.py
new file mode 100644
index 000000000..f3fd552f7
--- /dev/null
+++ b/skyvern/services/self_heal_reliability_service.py
@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from skyvern.forge import app
+from skyvern.schemas.self_heal import (
+ RELIABILITY_WINDOW,
+ HealEpisode,
+ RunHealGroup,
+ WorkflowReliability,
+ compute_workflow_reliability,
+)
+
+
+async def get_workflow_reliability(organization_id: str, workflow_permanent_id: str) -> WorkflowReliability:
+ run_ids = await app.DATABASE.self_heal.get_recent_terminal_workflow_run_ids(
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ limit=RELIABILITY_WINDOW,
+ )
+ episodes = await app.DATABASE.self_heal.get_heal_episodes_for_runs(
+ organization_id=organization_id,
+ workflow_run_ids=run_ids,
+ )
+
+ episodes_by_run_id: dict[str, list[HealEpisode]] = {run_id: [] for run_id in run_ids}
+ for episode in episodes:
+ if episode.workflow_run_id in episodes_by_run_id:
+ episodes_by_run_id[episode.workflow_run_id].append(episode)
+
+ groups = [RunHealGroup(workflow_run_id=run_id, episodes=episodes_by_run_id[run_id]) for run_id in run_ids]
+ return compute_workflow_reliability(groups)
diff --git a/tests/unit/forge/sdk/db/test_self_heal_repository.py b/tests/unit/forge/sdk/db/test_self_heal_repository.py
index ed8b12468..9f509b858 100644
--- a/tests/unit/forge/sdk/db/test_self_heal_repository.py
+++ b/tests/unit/forge/sdk/db/test_self_heal_repository.py
@@ -12,7 +12,7 @@ import pytest
import pytest_asyncio
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
-from skyvern.forge.sdk.db.models import Base, HealEpisodeModel, WorkflowHealProposalModel
+from skyvern.forge.sdk.db.models import Base, HealEpisodeModel, WorkflowHealProposalModel, WorkflowRunModel
from skyvern.forge.sdk.db.repositories.self_heal import SelfHealRepository
ORG_A = "o_aaaaaaaaaaaaaaa"
@@ -48,6 +48,31 @@ def _heal_episode_model(
return HealEpisodeModel(**model_kwargs)
+def _workflow_run_model(
+ *,
+ workflow_run_id: str,
+ created_at: datetime,
+ organization_id: str = ORG_A,
+ workflow_permanent_id: str = "wpid_1",
+ status: str = "completed",
+ parent_workflow_run_id: str | None = None,
+ copilot_session_id: str | None = None,
+ debug_session_id: str | None = None,
+) -> WorkflowRunModel:
+ return WorkflowRunModel(
+ workflow_run_id=workflow_run_id,
+ workflow_id="wf_1",
+ workflow_permanent_id=workflow_permanent_id,
+ organization_id=organization_id,
+ status=status,
+ created_at=created_at,
+ modified_at=created_at,
+ parent_workflow_run_id=parent_workflow_run_id,
+ copilot_session_id=copilot_session_id,
+ debug_session_id=debug_session_id,
+ )
+
+
@pytest_asyncio.fixture
async def repo_and_session() -> tuple:
engine = create_async_engine("sqlite+aiosqlite://")
@@ -55,7 +80,7 @@ async def repo_and_session() -> tuple:
await conn.run_sync(
lambda sync_conn: Base.metadata.create_all(
sync_conn,
- tables=[HealEpisodeModel.__table__, WorkflowHealProposalModel.__table__],
+ tables=[HealEpisodeModel.__table__, WorkflowHealProposalModel.__table__, WorkflowRunModel.__table__],
)
)
session_factory = async_sessionmaker(engine, expire_on_commit=False)
@@ -320,3 +345,135 @@ async def test_get_heal_episodes_for_run_uses_heal_episode_id_tiebreaker(repo_an
episodes = await repo.get_heal_episodes_for_run(organization_id=ORG_A, workflow_run_id="wr_tie")
assert [episode.heal_episode_id for episode in episodes] == ["he_001", "he_002", "he_003"]
+
+
+async def test_get_recent_terminal_workflow_run_ids_filters_orders_and_limits(repo_and_session) -> None:
+ repo, session_factory = repo_and_session
+ now = datetime(2026, 5, 1, 0, 0, 0)
+ async with session_factory() as session:
+ session.add_all(
+ [
+ _workflow_run_model(workflow_run_id="wr_old_completed", status="completed", created_at=now),
+ _workflow_run_model(
+ workflow_run_id="wr_mid_terminated",
+ status="terminated",
+ created_at=now + timedelta(minutes=2),
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_new_failed", status="failed", created_at=now + timedelta(minutes=3)
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_newest_timed_out",
+ status="timed_out",
+ created_at=now + timedelta(minutes=4),
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_running", status="running", created_at=now + timedelta(minutes=5)
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_other_org",
+ organization_id=ORG_B,
+ status="failed",
+ created_at=now + timedelta(minutes=6),
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_other_workflow",
+ workflow_permanent_id="wpid_other",
+ status="failed",
+ created_at=now + timedelta(minutes=7),
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_child",
+ status="completed",
+ created_at=now + timedelta(minutes=8),
+ parent_workflow_run_id="wr_old_completed",
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_copilot",
+ status="completed",
+ created_at=now + timedelta(minutes=9),
+ copilot_session_id="cs_1",
+ ),
+ _workflow_run_model(
+ workflow_run_id="wr_debug",
+ status="completed",
+ created_at=now + timedelta(minutes=10),
+ debug_session_id="ds_1",
+ ),
+ ]
+ )
+ await session.commit()
+
+ real_runs = await repo.get_recent_terminal_workflow_run_ids(
+ organization_id=ORG_A,
+ workflow_permanent_id="wpid_1",
+ limit=50,
+ )
+ assert real_runs == ["wr_newest_timed_out", "wr_new_failed", "wr_mid_terminated", "wr_old_completed"]
+
+ run_ids = await repo.get_recent_terminal_workflow_run_ids(
+ organization_id=ORG_A,
+ workflow_permanent_id="wpid_1",
+ limit=3,
+ )
+ assert run_ids == ["wr_newest_timed_out", "wr_new_failed", "wr_mid_terminated"]
+
+ bounded = await repo.get_recent_terminal_workflow_run_ids(
+ organization_id=ORG_A,
+ workflow_permanent_id="wpid_1",
+ limit=0,
+ )
+ assert bounded == ["wr_newest_timed_out"]
+
+
+async def test_get_heal_episodes_for_runs_batches_and_scopes_to_org(repo_and_session) -> None:
+ repo, session_factory = repo_and_session
+ now = datetime(2026, 6, 1, 0, 0, 0)
+
+ assert await repo.get_heal_episodes_for_runs(organization_id=ORG_A, workflow_run_ids=[]) == []
+
+ async with session_factory() as session:
+ session.add_all(
+ [
+ _heal_episode_model(
+ heal_episode_id="he_1",
+ workflow_run_id="wr_a",
+ workflow_run_block_id="wrb_a",
+ created_at=now + timedelta(minutes=2),
+ ),
+ _heal_episode_model(
+ heal_episode_id="he_2",
+ workflow_run_id="wr_b",
+ workflow_run_block_id="wrb_b",
+ created_at=now + timedelta(minutes=3),
+ ),
+ _heal_episode_model(
+ heal_episode_id="he_0",
+ workflow_run_id="wr_a",
+ workflow_run_block_id="wrb_c",
+ created_at=now + timedelta(minutes=1),
+ ),
+ _heal_episode_model(
+ heal_episode_id="he_other_org",
+ organization_id=ORG_B,
+ workflow_run_id="wr_a",
+ workflow_run_block_id="wrb_other",
+ created_at=now + timedelta(minutes=4),
+ ),
+ _heal_episode_model(
+ heal_episode_id="he_other_run",
+ workflow_run_id="wr_c",
+ workflow_run_block_id="wrb_other_run",
+ created_at=now + timedelta(minutes=5),
+ ),
+ ]
+ )
+ await session.commit()
+
+ episodes = await repo.get_heal_episodes_for_runs(
+ organization_id=ORG_A,
+ workflow_run_ids=["wr_b", "wr_a"],
+ )
+ assert [episode.heal_episode_id for episode in episodes] == ["he_0", "he_1", "he_2"]
diff --git a/tests/unit/forge/sdk/db/test_self_heal_schema.py b/tests/unit/forge/sdk/db/test_self_heal_schema.py
index f8554ae22..34a0b3206 100644
--- a/tests/unit/forge/sdk/db/test_self_heal_schema.py
+++ b/tests/unit/forge/sdk/db/test_self_heal_schema.py
@@ -15,6 +15,10 @@ from skyvern.schemas.self_heal import (
HealEpisodeView,
HealStatus,
OutputObligation,
+ ReliabilityState,
+ RunHealGroup,
+ compute_workflow_reliability,
+ reliability_state_transition,
resolve_block_outcome,
summarize_run_heals,
)
@@ -82,6 +86,35 @@ def _episode(
)
+def _run(workflow_run_id: str, episodes: list[HealEpisode]) -> RunHealGroup:
+ return RunHealGroup(workflow_run_id=workflow_run_id, episodes=episodes)
+
+
+def _episode_for_run(
+ *,
+ workflow_run_id: str,
+ block_label: str,
+ status: HealStatus,
+ output_obligation: OutputObligation | None = None,
+ engine: str = "code",
+) -> HealEpisode:
+ now = datetime(2026, 1, 1, 0, 0, 0)
+ return HealEpisode(
+ heal_episode_id=f"he_{workflow_run_id}_{block_label}",
+ organization_id="o_1",
+ workflow_permanent_id="wpid_1",
+ workflow_id="w_1",
+ workflow_run_id=workflow_run_id,
+ workflow_run_block_id=f"wrb_{workflow_run_id}_{block_label}",
+ block_label=block_label,
+ engine=engine,
+ status=status,
+ output_obligation=output_obligation,
+ created_at=now,
+ modified_at=now,
+ )
+
+
@pytest.mark.parametrize(
("episodes", "expected"),
[
@@ -136,3 +169,189 @@ def test_heal_episode_detail_exposes_only_sanitized_free_text_fields() -> None:
assert "block_code" not in fields
assert "block_prompt" not in fields
assert "failure_message" not in fields
+
+
+def test_compute_workflow_reliability_below_min_runs_is_unscored_healthy() -> None:
+ runs = [_run(f"wr_{idx}", []) for idx in range(9)]
+
+ reliability = compute_workflow_reliability(runs)
+
+ assert reliability.scored is False
+ assert reliability.state == ReliabilityState.healthy
+ assert reliability.window_runs == 9
+ assert reliability.healed_runs == 0
+ assert reliability.heal_rate == 0.0
+ assert reliability.consecutive_healed_runs == 0
+ assert reliability.floor_runs == 0
+ assert reliability.outcome_risk is False
+ assert reliability.outcome_risk_runs == 0
+
+
+def test_compute_workflow_reliability_watch_when_two_healed_in_window() -> None:
+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]
+ runs[5] = _run(
+ "wr_5",
+ [_episode_for_run(workflow_run_id="wr_5", block_label="block_5", status=HealStatus.fired_failed)],
+ )
+ runs[12] = _run(
+ "wr_12",
+ [_episode_for_run(workflow_run_id="wr_12", block_label="block_12", status=HealStatus.fired_completed)],
+ )
+
+ reliability = compute_workflow_reliability(runs)
+
+ assert reliability.scored is True
+ assert reliability.state == ReliabilityState.watch
+ assert reliability.healed_runs == 2
+ assert reliability.consecutive_healed_runs == 0
+ assert reliability.heal_rate == 0.1
+
+
+def test_compute_workflow_reliability_watch_when_two_consecutive_recent_healed() -> None:
+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]
+ runs[0] = _run(
+ "wr_0",
+ [_episode_for_run(workflow_run_id="wr_0", block_label="block_0", status=HealStatus.fired_unverified)],
+ )
+ runs[1] = _run(
+ "wr_1",
+ [_episode_for_run(workflow_run_id="wr_1", block_label="block_1", status=HealStatus.fired_failed)],
+ )
+
+ reliability = compute_workflow_reliability(runs)
+
+ assert reliability.state == ReliabilityState.watch
+ assert reliability.healed_runs == 2
+ assert reliability.consecutive_healed_runs == 2
+
+
+def test_compute_workflow_reliability_action_needed_when_three_recent_healed() -> None:
+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]
+ runs[0] = _run(
+ "wr_0",
+ [_episode_for_run(workflow_run_id="wr_0", block_label="block_0", status=HealStatus.fired_failed)],
+ )
+ runs[3] = _run(
+ "wr_3",
+ [_episode_for_run(workflow_run_id="wr_3", block_label="block_3", status=HealStatus.fired_completed)],
+ )
+ runs[7] = _run(
+ "wr_7",
+ [_episode_for_run(workflow_run_id="wr_7", block_label="block_7", status=HealStatus.fired_unverified)],
+ )
+
+ reliability = compute_workflow_reliability(runs)
+
+ assert reliability.state == ReliabilityState.action_needed
+ assert reliability.healed_runs == 3
+ assert reliability.heal_rate == 0.15
+
+
+def test_compute_workflow_reliability_action_needed_when_heal_rate_reaches_threshold() -> None:
+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]
+ for idx in (0, 5, 12, 18):
+ runs[idx] = _run(
+ f"wr_{idx}",
+ [_episode_for_run(workflow_run_id=f"wr_{idx}", block_label=f"block_{idx}", status=HealStatus.fired_failed)],
+ )
+
+ reliability = compute_workflow_reliability(runs)
+
+ assert reliability.state == ReliabilityState.action_needed
+ assert reliability.healed_runs == 4
+ assert reliability.heal_rate == 0.2
+
+
+def test_compute_workflow_reliability_action_needed_when_floor_runs_reach_threshold() -> None:
+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]
+ runs[2] = _run(
+ "wr_2",
+ [
+ _episode_for_run(
+ workflow_run_id="wr_2",
+ block_label="block_2",
+ status=HealStatus.skipped,
+ engine="floor",
+ )
+ ],
+ )
+ runs[13] = _run(
+ "wr_13",
+ [
+ _episode_for_run(
+ workflow_run_id="wr_13",
+ block_label="block_13",
+ status=HealStatus.skipped,
+ engine="floor",
+ )
+ ],
+ )
+
+ reliability = compute_workflow_reliability(runs)
+
+ assert reliability.state == ReliabilityState.action_needed
+ assert reliability.floor_runs == 2
+ assert reliability.healed_runs == 0
+
+
+def test_compute_workflow_reliability_sets_outcome_risk_from_recent_runs() -> None:
+ runs = [_run(f"wr_{idx}", []) for idx in range(10)]
+ runs[0] = _run(
+ "wr_0",
+ [
+ _episode_for_run(
+ workflow_run_id="wr_0",
+ block_label="invoice_submit",
+ status=HealStatus.fired_unverified,
+ output_obligation=OutputObligation.observed,
+ )
+ ],
+ )
+
+ reliability = compute_workflow_reliability(runs)
+
+ assert reliability.outcome_risk is True
+ assert reliability.outcome_risk_runs == 1
+ assert reliability.state == ReliabilityState.healthy
+
+
+def test_compute_workflow_reliability_consecutive_streak_stops_at_first_unhealed() -> None:
+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]
+ runs[0] = _run(
+ "wr_0",
+ [_episode_for_run(workflow_run_id="wr_0", block_label="block_0", status=HealStatus.fired_completed)],
+ )
+ runs[1] = _run(
+ "wr_1",
+ [_episode_for_run(workflow_run_id="wr_1", block_label="block_1", status=HealStatus.fired_failed)],
+ )
+ runs[3] = _run(
+ "wr_3",
+ [_episode_for_run(workflow_run_id="wr_3", block_label="block_3", status=HealStatus.fired_failed)],
+ )
+
+ reliability = compute_workflow_reliability(runs)
+
+ assert reliability.consecutive_healed_runs == 2
+ assert reliability.healed_runs == 3
+ assert reliability.state == ReliabilityState.action_needed
+
+
+def test_compute_workflow_reliability_heal_rate_math() -> None:
+ runs = [
+ _run("wr_0", [_episode_for_run(workflow_run_id="wr_0", block_label="block_0", status=HealStatus.fired_failed)]),
+ _run("wr_1", []),
+ _run("wr_2", []),
+ ]
+
+ reliability = compute_workflow_reliability(runs)
+
+ assert reliability.healed_runs == 1
+ assert reliability.window_runs == 3
+ assert reliability.heal_rate == pytest.approx(1 / 3)
+
+
+def test_reliability_state_transition() -> None:
+ assert reliability_state_transition(None, ReliabilityState.healthy) is True
+ assert reliability_state_transition(ReliabilityState.watch, ReliabilityState.watch) is False
+ assert reliability_state_transition(ReliabilityState.watch, ReliabilityState.action_needed) is True
diff --git a/tests/unit/forge/sdk/routes/test_self_heal_routes.py b/tests/unit/forge/sdk/routes/test_self_heal_routes.py
index c2cd9fde2..a00ab5b53 100644
--- a/tests/unit/forge/sdk/routes/test_self_heal_routes.py
+++ b/tests/unit/forge/sdk/routes/test_self_heal_routes.py
@@ -7,11 +7,18 @@ from unittest.mock import AsyncMock
from fastapi import FastAPI
from fastapi.testclient import TestClient
-import skyvern.forge.sdk.routes.self_heal # noqa: F401
+import skyvern.forge.sdk.routes.self_heal as self_heal_routes
from skyvern.forge import app as forge_app
from skyvern.forge.sdk.routes import routers as routers_module
from skyvern.forge.sdk.services import org_auth_service
-from skyvern.schemas.self_heal import HealEpisode, HealEpisodeView, HealStatus, OutputObligation
+from skyvern.schemas.self_heal import (
+ HealEpisode,
+ HealEpisodeView,
+ HealStatus,
+ OutputObligation,
+ ReliabilityState,
+ WorkflowReliability,
+)
ORG_ID = "org_self_heal"
@@ -43,23 +50,25 @@ def _episode(
)
-def _build_client(monkeypatch) -> tuple[TestClient, AsyncMock, AsyncMock]:
+def _build_client(monkeypatch) -> tuple[TestClient, AsyncMock, AsyncMock, AsyncMock]:
async def _fake_org() -> SimpleNamespace:
return SimpleNamespace(organization_id=ORG_ID)
get_for_workflow = AsyncMock()
get_for_run = AsyncMock()
+ get_reliability = AsyncMock()
monkeypatch.setattr(forge_app.DATABASE.self_heal, "get_heal_episodes_for_workflow", get_for_workflow)
monkeypatch.setattr(forge_app.DATABASE.self_heal, "get_heal_episodes_for_run", get_for_run)
+ monkeypatch.setattr(self_heal_routes, "get_workflow_reliability", get_reliability)
fastapi_app = FastAPI()
fastapi_app.dependency_overrides[org_auth_service.get_current_org] = _fake_org
fastapi_app.include_router(routers_module.base_router, prefix="/v1")
- return TestClient(fastapi_app), get_for_workflow, get_for_run
+ return TestClient(fastapi_app), get_for_workflow, get_for_run, get_reliability
def test_get_workflow_heal_episodes_filters_to_caller_org_and_view_shape(monkeypatch) -> None:
- client, get_for_workflow, _ = _build_client(monkeypatch)
+ client, get_for_workflow, _, _ = _build_client(monkeypatch)
async def _repo_call(**kwargs):
assert kwargs["organization_id"] == ORG_ID
@@ -99,7 +108,7 @@ def test_get_workflow_heal_episodes_filters_to_caller_org_and_view_shape(monkeyp
def test_get_workflow_heal_episodes_invalid_status_returns_422(monkeypatch) -> None:
- client, get_for_workflow, _ = _build_client(monkeypatch)
+ client, get_for_workflow, _, _ = _build_client(monkeypatch)
response = client.get("/v1/workflows/wpid_target/heal_episodes", params={"status": "invalid_status"})
@@ -108,7 +117,7 @@ def test_get_workflow_heal_episodes_invalid_status_returns_422(monkeypatch) -> N
def test_get_run_heal_episodes_returns_episodes_and_summary(monkeypatch) -> None:
- client, _, get_for_run = _build_client(monkeypatch)
+ client, _, get_for_run, _ = _build_client(monkeypatch)
get_for_run.return_value = [
_episode(
heal_episode_id="he_1",
@@ -141,3 +150,35 @@ def test_get_run_heal_episodes_returns_episodes_and_summary(monkeypatch) -> None
assert body["summary"]["blocks_healed"] == 1
assert body["summary"]["blocks_with_heal_attempt"] == 2
assert body["summary"]["blocks_outcome_risk"] == ["block_b"]
+
+
+def test_get_workflow_reliability_is_org_scoped(monkeypatch) -> None:
+ client, _, _, get_reliability = _build_client(monkeypatch)
+ get_reliability.return_value = WorkflowReliability(
+ state=ReliabilityState.watch,
+ outcome_risk=True,
+ scored=True,
+ window_runs=20,
+ healed_runs=2,
+ heal_rate=0.1,
+ consecutive_healed_runs=1,
+ floor_runs=0,
+ outcome_risk_runs=1,
+ )
+
+ response = client.get("/v1/workflows/wpid_target/reliability")
+
+ assert response.status_code == 200
+ body = response.json()
+ assert body == {
+ "state": "watch",
+ "outcome_risk": True,
+ "scored": True,
+ "window_runs": 20,
+ "healed_runs": 2,
+ "heal_rate": 0.1,
+ "consecutive_healed_runs": 1,
+ "floor_runs": 0,
+ "outcome_risk_runs": 1,
+ }
+ get_reliability.assert_awaited_once_with(ORG_ID, "wpid_target")