Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-12215): self-heal reliability aggregation service (runs × episodes) (#7415)
pedrohsdbedfe565fa3a5
diff --git a/skyvern/forge/sdk/db/repositories/self_heal.py b/skyvern/forge/sdk/db/repositories/self_heal.pyindex aefe1ffed..67bae738e 100644--- a/skyvern/forge/sdk/db/repositories/self_heal.py+++ b/skyvern/forge/sdk/db/repositories/self_heal.py@@ -7,7 +7,7 @@ from sqlalchemy import select, updatefrom skyvern.forge.sdk.db._error_handling import db_operationfrom skyvern.forge.sdk.db.base_repository import BaseRepositoryfrom skyvern.forge.sdk.db.datetime_utils import naive_utc_now-from skyvern.forge.sdk.db.models import HealEpisodeModel, WorkflowHealProposalModel+from skyvern.forge.sdk.db.models import HealEpisodeModel, WorkflowHealProposalModel, WorkflowRunModelfrom skyvern.forge.sdk.utils.sanitization import sanitize_postgres_textfrom skyvern.schemas.self_heal import HealEpisode, HealSkipReason, HealStatus, OutputObligation, WorkflowHealProposal@@ -143,6 +143,43 @@ class SelfHealRepository(BaseRepository):).all()return [HealEpisode.model_validate(episode) for episode in episodes]+ @db_operation("get_recent_terminal_workflow_run_ids")+ async def get_recent_terminal_workflow_run_ids(+ self,+ organization_id: str,+ workflow_permanent_id: str,+ limit: int = 20,+ ) -> list[str]:+ terminal_statuses = ("completed", "failed", "terminated", "timed_out")+ bounded_limit = max(1, min(limit, 100))+ async with self.Session() as session:+ query = (+ select(WorkflowRunModel.workflow_run_id)+ .where(+ WorkflowRunModel.organization_id == organization_id,+ WorkflowRunModel.workflow_permanent_id == workflow_permanent_id,+ WorkflowRunModel.status.in_(terminal_statuses),+ WorkflowRunModel.parent_workflow_run_id.is_(None),+ WorkflowRunModel.copilot_session_id.is_(None),+ WorkflowRunModel.debug_session_id.is_(None),+ )+ .order_by(WorkflowRunModel.created_at.desc(), WorkflowRunModel.workflow_run_id.desc())+ .limit(bounded_limit)+ )+ return list((await session.scalars(query)).all())++ @db_operation("get_heal_episodes_for_runs")+ async def get_heal_episodes_for_runs(self, organization_id: str, workflow_run_ids: list[str]) -> list[HealEpisode]:+ if not workflow_run_ids:+ return []+ async with self.Session() as session:+ query = select(HealEpisodeModel).where(+ HealEpisodeModel.organization_id == organization_id,+ HealEpisodeModel.workflow_run_id.in_(workflow_run_ids),+ )+ episodes = (await session.scalars(query.order_by(HealEpisodeModel.created_at.asc()))).all()+ return [HealEpisode.model_validate(episode) for episode in episodes]+@db_operation("create_heal_proposal")async def create_heal_proposal(self,diff --git a/skyvern/forge/sdk/routes/self_heal.py b/skyvern/forge/sdk/routes/self_heal.pyindex ec52e23ac..d4715f543 100644--- a/skyvern/forge/sdk/routes/self_heal.py+++ b/skyvern/forge/sdk/routes/self_heal.py@@ -6,7 +6,14 @@ from skyvern.forge import appfrom skyvern.forge.sdk.routes.routers import base_routerfrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.services import org_auth_service-from skyvern.schemas.self_heal import HealEpisodeView, HealStatus, RunHealSummary, summarize_run_heals+from skyvern.schemas.self_heal import (+ HealEpisodeView,+ HealStatus,+ RunHealSummary,+ WorkflowReliability,+ summarize_run_heals,+)+from skyvern.services.self_heal_reliability_service import get_workflow_reliabilityLOG = structlog.get_logger()@@ -57,3 +64,15 @@ async def get_run_heal_episodes(episodes=[HealEpisodeView.from_episode(episode) for episode in episodes],summary=summarize_run_heals(episodes),)+++@base_router.get(+ "/workflows/{workflow_permanent_id}/reliability",+ response_model=WorkflowReliability,+ include_in_schema=False,+)+async def get_workflow_reliability_route(+ workflow_permanent_id: str,+ organization: Organization = Depends(org_auth_service.get_current_org),+) -> WorkflowReliability:+ return await get_workflow_reliability(organization.organization_id, workflow_permanent_id)diff --git a/skyvern/schemas/self_heal.py b/skyvern/schemas/self_heal.pyindex e5b22c216..da81758a5 100644--- a/skyvern/schemas/self_heal.py+++ b/skyvern/schemas/self_heal.py@@ -33,6 +33,16 @@ class OutputObligation(StrEnum):observed = "observed"+class ReliabilityState(StrEnum):+ healthy = "healthy"+ watch = "watch"+ action_needed = "action_needed"+++RELIABILITY_WINDOW = 20+RELIABILITY_MIN_RUNS = 10++@dataclass(frozen=True)class HealClassification:healable: bool@@ -157,6 +167,23 @@ class RunHealSummary(BaseModel):blocks_with_heal_attempt: int+class RunHealGroup(BaseModel):+ workflow_run_id: str+ episodes: list[HealEpisode]+++class WorkflowReliability(BaseModel):+ state: ReliabilityState+ outcome_risk: bool+ scored: bool+ window_runs: int+ healed_runs: int+ heal_rate: float+ consecutive_healed_runs: int+ floor_runs: int+ outcome_risk_runs: int++def summarize_run_heals(episodes: list[HealEpisode]) -> RunHealSummary:episodes_by_block: dict[str, list[HealEpisode]] = {}for episode in episodes:@@ -184,6 +211,59 @@ def summarize_run_heals(episodes: list[HealEpisode]) -> RunHealSummary:)+def compute_workflow_reliability(runs: list[RunHealGroup]) -> WorkflowReliability:+ run_flags: list[tuple[bool, bool, bool]] = []+ for run in runs:+ healed = any(+ episode.status in {HealStatus.fired_completed, HealStatus.fired_unverified, HealStatus.fired_failed}+ for episode in run.episodes+ )+ floor = any(episode.engine == "floor" for episode in run.episodes)+ outcome_risk = len(summarize_run_heals(run.episodes).blocks_outcome_risk) > 0+ run_flags.append((healed, floor, outcome_risk))++ window_runs = len(runs)+ healed_runs = sum(healed for healed, _, _ in run_flags)+ heal_rate = healed_runs / window_runs if window_runs else 0.0++ consecutive_healed_runs = 0+ for healed, _, _ in run_flags:+ if not healed:+ break+ consecutive_healed_runs += 1++ floor_runs = sum(floor for _, floor, _ in run_flags)+ recent_run_flags = run_flags[:10]+ recent_healed_runs = sum(healed for healed, _, _ in recent_run_flags)+ outcome_risk_runs = sum(outcome_risk for _, _, outcome_risk in recent_run_flags)++ scored = window_runs >= RELIABILITY_MIN_RUNS+ if not scored:+ state = ReliabilityState.healthy+ elif recent_healed_runs >= 3 or heal_rate >= 0.20 or floor_runs >= 2:+ state = ReliabilityState.action_needed+ elif healed_runs >= 2 or consecutive_healed_runs >= 2:+ state = ReliabilityState.watch+ else:+ state = ReliabilityState.healthy++ return WorkflowReliability(+ state=state,+ outcome_risk=outcome_risk_runs > 0,+ scored=scored,+ window_runs=window_runs,+ healed_runs=healed_runs,+ heal_rate=heal_rate,+ consecutive_healed_runs=consecutive_healed_runs,+ floor_runs=floor_runs,+ outcome_risk_runs=outcome_risk_runs,+ )+++def reliability_state_transition(previous: ReliabilityState | None, current: ReliabilityState) -> bool:+ return previous is None or previous != current++class WorkflowHealProposal(BaseModel):model_config = ConfigDict(from_attributes=True)diff --git a/skyvern/services/self_heal_reliability_service.py b/skyvern/services/self_heal_reliability_service.pynew file mode 100644index 000000000..f3fd552f7--- /dev/null+++ b/skyvern/services/self_heal_reliability_service.py@@ -0,0 +1,30 @@+from __future__ import annotations++from skyvern.forge import app+from skyvern.schemas.self_heal import (+ RELIABILITY_WINDOW,+ HealEpisode,+ RunHealGroup,+ WorkflowReliability,+ compute_workflow_reliability,+)+++async def get_workflow_reliability(organization_id: str, workflow_permanent_id: str) -> WorkflowReliability:+ run_ids = await app.DATABASE.self_heal.get_recent_terminal_workflow_run_ids(+ organization_id=organization_id,+ workflow_permanent_id=workflow_permanent_id,+ limit=RELIABILITY_WINDOW,+ )+ episodes = await app.DATABASE.self_heal.get_heal_episodes_for_runs(+ organization_id=organization_id,+ workflow_run_ids=run_ids,+ )++ episodes_by_run_id: dict[str, list[HealEpisode]] = {run_id: [] for run_id in run_ids}+ for episode in episodes:+ if episode.workflow_run_id in episodes_by_run_id:+ episodes_by_run_id[episode.workflow_run_id].append(episode)++ groups = [RunHealGroup(workflow_run_id=run_id, episodes=episodes_by_run_id[run_id]) for run_id in run_ids]+ return compute_workflow_reliability(groups)diff --git a/tests/unit/forge/sdk/db/test_self_heal_repository.py b/tests/unit/forge/sdk/db/test_self_heal_repository.pyindex ed8b12468..9f509b858 100644--- a/tests/unit/forge/sdk/db/test_self_heal_repository.py+++ b/tests/unit/forge/sdk/db/test_self_heal_repository.py@@ -12,7 +12,7 @@ import pytestimport pytest_asynciofrom sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine-from skyvern.forge.sdk.db.models import Base, HealEpisodeModel, WorkflowHealProposalModel+from skyvern.forge.sdk.db.models import Base, HealEpisodeModel, WorkflowHealProposalModel, WorkflowRunModelfrom skyvern.forge.sdk.db.repositories.self_heal import SelfHealRepositoryORG_A = "o_aaaaaaaaaaaaaaa"@@ -48,6 +48,31 @@ def _heal_episode_model(return HealEpisodeModel(**model_kwargs)+def _workflow_run_model(+ *,+ workflow_run_id: str,+ created_at: datetime,+ organization_id: str = ORG_A,+ workflow_permanent_id: str = "wpid_1",+ status: str = "completed",+ parent_workflow_run_id: str | None = None,+ copilot_session_id: str | None = None,+ debug_session_id: str | None = None,+) -> WorkflowRunModel:+ return WorkflowRunModel(+ workflow_run_id=workflow_run_id,+ workflow_id="wf_1",+ workflow_permanent_id=workflow_permanent_id,+ organization_id=organization_id,+ status=status,+ created_at=created_at,+ modified_at=created_at,+ parent_workflow_run_id=parent_workflow_run_id,+ copilot_session_id=copilot_session_id,+ debug_session_id=debug_session_id,+ )++@pytest_asyncio.fixtureasync def repo_and_session() -> tuple:engine = create_async_engine("sqlite+aiosqlite://")@@ -55,7 +80,7 @@ async def repo_and_session() -> tuple:await conn.run_sync(lambda sync_conn: Base.metadata.create_all(sync_conn,- tables=[HealEpisodeModel.__table__, WorkflowHealProposalModel.__table__],+ tables=[HealEpisodeModel.__table__, WorkflowHealProposalModel.__table__, WorkflowRunModel.__table__],))session_factory = async_sessionmaker(engine, expire_on_commit=False)@@ -320,3 +345,135 @@ async def test_get_heal_episodes_for_run_uses_heal_episode_id_tiebreaker(repo_anepisodes = await repo.get_heal_episodes_for_run(organization_id=ORG_A, workflow_run_id="wr_tie")assert [episode.heal_episode_id for episode in episodes] == ["he_001", "he_002", "he_003"]+++async def test_get_recent_terminal_workflow_run_ids_filters_orders_and_limits(repo_and_session) -> None:+ repo, session_factory = repo_and_session+ now = datetime(2026, 5, 1, 0, 0, 0)+ async with session_factory() as session:+ session.add_all(+ [+ _workflow_run_model(workflow_run_id="wr_old_completed", status="completed", created_at=now),+ _workflow_run_model(+ workflow_run_id="wr_mid_terminated",+ status="terminated",+ created_at=now + timedelta(minutes=2),+ ),+ _workflow_run_model(+ workflow_run_id="wr_new_failed", status="failed", created_at=now + timedelta(minutes=3)+ ),+ _workflow_run_model(+ workflow_run_id="wr_newest_timed_out",+ status="timed_out",+ created_at=now + timedelta(minutes=4),+ ),+ _workflow_run_model(+ workflow_run_id="wr_running", status="running", created_at=now + timedelta(minutes=5)+ ),+ _workflow_run_model(+ workflow_run_id="wr_other_org",+ organization_id=ORG_B,+ status="failed",+ created_at=now + timedelta(minutes=6),+ ),+ _workflow_run_model(+ workflow_run_id="wr_other_workflow",+ workflow_permanent_id="wpid_other",+ status="failed",+ created_at=now + timedelta(minutes=7),+ ),+ _workflow_run_model(+ workflow_run_id="wr_child",+ status="completed",+ created_at=now + timedelta(minutes=8),+ parent_workflow_run_id="wr_old_completed",+ ),+ _workflow_run_model(+ workflow_run_id="wr_copilot",+ status="completed",+ created_at=now + timedelta(minutes=9),+ copilot_session_id="cs_1",+ ),+ _workflow_run_model(+ workflow_run_id="wr_debug",+ status="completed",+ created_at=now + timedelta(minutes=10),+ debug_session_id="ds_1",+ ),+ ]+ )+ await session.commit()++ real_runs = await repo.get_recent_terminal_workflow_run_ids(+ organization_id=ORG_A,+ workflow_permanent_id="wpid_1",+ limit=50,+ )+ assert real_runs == ["wr_newest_timed_out", "wr_new_failed", "wr_mid_terminated", "wr_old_completed"]++ run_ids = await repo.get_recent_terminal_workflow_run_ids(+ organization_id=ORG_A,+ workflow_permanent_id="wpid_1",+ limit=3,+ )+ assert run_ids == ["wr_newest_timed_out", "wr_new_failed", "wr_mid_terminated"]++ bounded = await repo.get_recent_terminal_workflow_run_ids(+ organization_id=ORG_A,+ workflow_permanent_id="wpid_1",+ limit=0,+ )+ assert bounded == ["wr_newest_timed_out"]+++async def test_get_heal_episodes_for_runs_batches_and_scopes_to_org(repo_and_session) -> None:+ repo, session_factory = repo_and_session+ now = datetime(2026, 6, 1, 0, 0, 0)++ assert await repo.get_heal_episodes_for_runs(organization_id=ORG_A, workflow_run_ids=[]) == []++ async with session_factory() as session:+ session.add_all(+ [+ _heal_episode_model(+ heal_episode_id="he_1",+ workflow_run_id="wr_a",+ workflow_run_block_id="wrb_a",+ created_at=now + timedelta(minutes=2),+ ),+ _heal_episode_model(+ heal_episode_id="he_2",+ workflow_run_id="wr_b",+ workflow_run_block_id="wrb_b",+ created_at=now + timedelta(minutes=3),+ ),+ _heal_episode_model(+ heal_episode_id="he_0",+ workflow_run_id="wr_a",+ workflow_run_block_id="wrb_c",+ created_at=now + timedelta(minutes=1),+ ),+ _heal_episode_model(+ heal_episode_id="he_other_org",+ organization_id=ORG_B,+ workflow_run_id="wr_a",+ workflow_run_block_id="wrb_other",+ created_at=now + timedelta(minutes=4),+ ),+ _heal_episode_model(+ heal_episode_id="he_other_run",+ workflow_run_id="wr_c",+ workflow_run_block_id="wrb_other_run",+ created_at=now + timedelta(minutes=5),+ ),+ ]+ )+ await session.commit()++ episodes = await repo.get_heal_episodes_for_runs(+ organization_id=ORG_A,+ workflow_run_ids=["wr_b", "wr_a"],+ )+ assert [episode.heal_episode_id for episode in episodes] == ["he_0", "he_1", "he_2"]diff --git a/tests/unit/forge/sdk/db/test_self_heal_schema.py b/tests/unit/forge/sdk/db/test_self_heal_schema.pyindex f8554ae22..34a0b3206 100644--- a/tests/unit/forge/sdk/db/test_self_heal_schema.py+++ b/tests/unit/forge/sdk/db/test_self_heal_schema.py@@ -15,6 +15,10 @@ from skyvern.schemas.self_heal import (HealEpisodeView,HealStatus,OutputObligation,+ ReliabilityState,+ RunHealGroup,+ compute_workflow_reliability,+ reliability_state_transition,resolve_block_outcome,summarize_run_heals,)@@ -82,6 +86,35 @@ def _episode()+def _run(workflow_run_id: str, episodes: list[HealEpisode]) -> RunHealGroup:+ return RunHealGroup(workflow_run_id=workflow_run_id, episodes=episodes)+++def _episode_for_run(+ *,+ workflow_run_id: str,+ block_label: str,+ status: HealStatus,+ output_obligation: OutputObligation | None = None,+ engine: str = "code",+) -> HealEpisode:+ now = datetime(2026, 1, 1, 0, 0, 0)+ return HealEpisode(+ heal_episode_id=f"he_{workflow_run_id}_{block_label}",+ organization_id="o_1",+ workflow_permanent_id="wpid_1",+ workflow_id="w_1",+ workflow_run_id=workflow_run_id,+ workflow_run_block_id=f"wrb_{workflow_run_id}_{block_label}",+ block_label=block_label,+ engine=engine,+ status=status,+ output_obligation=output_obligation,+ created_at=now,+ modified_at=now,+ )++@pytest.mark.parametrize(("episodes", "expected"),[@@ -136,3 +169,189 @@ def test_heal_episode_detail_exposes_only_sanitized_free_text_fields() -> None:assert "block_code" not in fieldsassert "block_prompt" not in fieldsassert "failure_message" not in fields+++def test_compute_workflow_reliability_below_min_runs_is_unscored_healthy() -> None:+ runs = [_run(f"wr_{idx}", []) for idx in range(9)]++ reliability = compute_workflow_reliability(runs)++ assert reliability.scored is False+ assert reliability.state == ReliabilityState.healthy+ assert reliability.window_runs == 9+ assert reliability.healed_runs == 0+ assert reliability.heal_rate == 0.0+ assert reliability.consecutive_healed_runs == 0+ assert reliability.floor_runs == 0+ assert reliability.outcome_risk is False+ assert reliability.outcome_risk_runs == 0+++def test_compute_workflow_reliability_watch_when_two_healed_in_window() -> None:+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]+ runs[5] = _run(+ "wr_5",+ [_episode_for_run(workflow_run_id="wr_5", block_label="block_5", status=HealStatus.fired_failed)],+ )+ runs[12] = _run(+ "wr_12",+ [_episode_for_run(workflow_run_id="wr_12", block_label="block_12", status=HealStatus.fired_completed)],+ )++ reliability = compute_workflow_reliability(runs)++ assert reliability.scored is True+ assert reliability.state == ReliabilityState.watch+ assert reliability.healed_runs == 2+ assert reliability.consecutive_healed_runs == 0+ assert reliability.heal_rate == 0.1+++def test_compute_workflow_reliability_watch_when_two_consecutive_recent_healed() -> None:+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]+ runs[0] = _run(+ "wr_0",+ [_episode_for_run(workflow_run_id="wr_0", block_label="block_0", status=HealStatus.fired_unverified)],+ )+ runs[1] = _run(+ "wr_1",+ [_episode_for_run(workflow_run_id="wr_1", block_label="block_1", status=HealStatus.fired_failed)],+ )++ reliability = compute_workflow_reliability(runs)++ assert reliability.state == ReliabilityState.watch+ assert reliability.healed_runs == 2+ assert reliability.consecutive_healed_runs == 2+++def test_compute_workflow_reliability_action_needed_when_three_recent_healed() -> None:+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]+ runs[0] = _run(+ "wr_0",+ [_episode_for_run(workflow_run_id="wr_0", block_label="block_0", status=HealStatus.fired_failed)],+ )+ runs[3] = _run(+ "wr_3",+ [_episode_for_run(workflow_run_id="wr_3", block_label="block_3", status=HealStatus.fired_completed)],+ )+ runs[7] = _run(+ "wr_7",+ [_episode_for_run(workflow_run_id="wr_7", block_label="block_7", status=HealStatus.fired_unverified)],+ )++ reliability = compute_workflow_reliability(runs)++ assert reliability.state == ReliabilityState.action_needed+ assert reliability.healed_runs == 3+ assert reliability.heal_rate == 0.15+++def test_compute_workflow_reliability_action_needed_when_heal_rate_reaches_threshold() -> None:+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]+ for idx in (0, 5, 12, 18):+ runs[idx] = _run(+ f"wr_{idx}",+ [_episode_for_run(workflow_run_id=f"wr_{idx}", block_label=f"block_{idx}", status=HealStatus.fired_failed)],+ )++ reliability = compute_workflow_reliability(runs)++ assert reliability.state == ReliabilityState.action_needed+ assert reliability.healed_runs == 4+ assert reliability.heal_rate == 0.2+++def test_compute_workflow_reliability_action_needed_when_floor_runs_reach_threshold() -> None:+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]+ runs[2] = _run(+ "wr_2",+ [+ _episode_for_run(+ workflow_run_id="wr_2",+ block_label="block_2",+ status=HealStatus.skipped,+ engine="floor",+ )+ ],+ )+ runs[13] = _run(+ "wr_13",+ [+ _episode_for_run(+ workflow_run_id="wr_13",+ block_label="block_13",+ status=HealStatus.skipped,+ engine="floor",+ )+ ],+ )++ reliability = compute_workflow_reliability(runs)++ assert reliability.state == ReliabilityState.action_needed+ assert reliability.floor_runs == 2+ assert reliability.healed_runs == 0+++def test_compute_workflow_reliability_sets_outcome_risk_from_recent_runs() -> None:+ runs = [_run(f"wr_{idx}", []) for idx in range(10)]+ runs[0] = _run(+ "wr_0",+ [+ _episode_for_run(+ workflow_run_id="wr_0",+ block_label="invoice_submit",+ status=HealStatus.fired_unverified,+ output_obligation=OutputObligation.observed,+ )+ ],+ )++ reliability = compute_workflow_reliability(runs)++ assert reliability.outcome_risk is True+ assert reliability.outcome_risk_runs == 1+ assert reliability.state == ReliabilityState.healthy+++def test_compute_workflow_reliability_consecutive_streak_stops_at_first_unhealed() -> None:+ runs = [_run(f"wr_{idx}", []) for idx in range(20)]+ runs[0] = _run(+ "wr_0",+ [_episode_for_run(workflow_run_id="wr_0", block_label="block_0", status=HealStatus.fired_completed)],+ )+ runs[1] = _run(+ "wr_1",+ [_episode_for_run(workflow_run_id="wr_1", block_label="block_1", status=HealStatus.fired_failed)],+ )+ runs[3] = _run(+ "wr_3",+ [_episode_for_run(workflow_run_id="wr_3", block_label="block_3", status=HealStatus.fired_failed)],+ )++ reliability = compute_workflow_reliability(runs)++ assert reliability.consecutive_healed_runs == 2+ assert reliability.healed_runs == 3+ assert reliability.state == ReliabilityState.action_needed+++def test_compute_workflow_reliability_heal_rate_math() -> None:+ runs = [+ _run("wr_0", [_episode_for_run(workflow_run_id="wr_0", block_label="block_0", status=HealStatus.fired_failed)]),+ _run("wr_1", []),+ _run("wr_2", []),+ ]++ reliability = compute_workflow_reliability(runs)++ assert reliability.healed_runs == 1+ assert reliability.window_runs == 3+ assert reliability.heal_rate == pytest.approx(1 / 3)+++def test_reliability_state_transition() -> None:+ assert reliability_state_transition(None, ReliabilityState.healthy) is True+ assert reliability_state_transition(ReliabilityState.watch, ReliabilityState.watch) is False+ assert reliability_state_transition(ReliabilityState.watch, ReliabilityState.action_needed) is Truediff --git a/tests/unit/forge/sdk/routes/test_self_heal_routes.py b/tests/unit/forge/sdk/routes/test_self_heal_routes.pyindex c2cd9fde2..a00ab5b53 100644--- a/tests/unit/forge/sdk/routes/test_self_heal_routes.py+++ b/tests/unit/forge/sdk/routes/test_self_heal_routes.py@@ -7,11 +7,18 @@ from unittest.mock import AsyncMockfrom fastapi import FastAPIfrom fastapi.testclient import TestClient-import skyvern.forge.sdk.routes.self_heal # noqa: F401+import skyvern.forge.sdk.routes.self_heal as self_heal_routesfrom skyvern.forge import app as forge_appfrom skyvern.forge.sdk.routes import routers as routers_modulefrom skyvern.forge.sdk.services import org_auth_service-from skyvern.schemas.self_heal import HealEpisode, HealEpisodeView, HealStatus, OutputObligation+from skyvern.schemas.self_heal import (+ HealEpisode,+ HealEpisodeView,+ HealStatus,+ OutputObligation,+ ReliabilityState,+ WorkflowReliability,+)ORG_ID = "org_self_heal"@@ -43,23 +50,25 @@ def _episode()-def _build_client(monkeypatch) -> tuple[TestClient, AsyncMock, AsyncMock]:+def _build_client(monkeypatch) -> tuple[TestClient, AsyncMock, AsyncMock, AsyncMock]:async def _fake_org() -> SimpleNamespace:return SimpleNamespace(organization_id=ORG_ID)get_for_workflow = AsyncMock()get_for_run = AsyncMock()+ get_reliability = AsyncMock()monkeypatch.setattr(forge_app.DATABASE.self_heal, "get_heal_episodes_for_workflow", get_for_workflow)monkeypatch.setattr(forge_app.DATABASE.self_heal, "get_heal_episodes_for_run", get_for_run)+ monkeypatch.setattr(self_heal_routes, "get_workflow_reliability", get_reliability)fastapi_app = FastAPI()fastapi_app.dependency_overrides[org_auth_service.get_current_org] = _fake_orgfastapi_app.include_router(routers_module.base_router, prefix="/v1")- return TestClient(fastapi_app), get_for_workflow, get_for_run+ return TestClient(fastapi_app), get_for_workflow, get_for_run, get_reliabilitydef test_get_workflow_heal_episodes_filters_to_caller_org_and_view_shape(monkeypatch) -> None:- client, get_for_workflow, _ = _build_client(monkeypatch)+ client, get_for_workflow, _, _ = _build_client(monkeypatch)async def _repo_call(**kwargs):assert kwargs["organization_id"] == ORG_ID@@ -99,7 +108,7 @@ def test_get_workflow_heal_episodes_filters_to_caller_org_and_view_shape(monkeypdef test_get_workflow_heal_episodes_invalid_status_returns_422(monkeypatch) -> None:- client, get_for_workflow, _ = _build_client(monkeypatch)+ client, get_for_workflow, _, _ = _build_client(monkeypatch)response = client.get("/v1/workflows/wpid_target/heal_episodes", params={"status": "invalid_status"})@@ -108,7 +117,7 @@ def test_get_workflow_heal_episodes_invalid_status_returns_422(monkeypatch) -> Ndef test_get_run_heal_episodes_returns_episodes_and_summary(monkeypatch) -> None:- client, _, get_for_run = _build_client(monkeypatch)+ client, _, get_for_run, _ = _build_client(monkeypatch)get_for_run.return_value = [_episode(heal_episode_id="he_1",@@ -141,3 +150,35 @@ def test_get_run_heal_episodes_returns_episodes_and_summary(monkeypatch) -> Noneassert body["summary"]["blocks_healed"] == 1assert body["summary"]["blocks_with_heal_attempt"] == 2assert body["summary"]["blocks_outcome_risk"] == ["block_b"]+++def test_get_workflow_reliability_is_org_scoped(monkeypatch) -> None:+ client, _, _, get_reliability = _build_client(monkeypatch)+ get_reliability.return_value = WorkflowReliability(+ state=ReliabilityState.watch,+ outcome_risk=True,+ scored=True,+ window_runs=20,+ healed_runs=2,+ heal_rate=0.1,+ consecutive_healed_runs=1,+ floor_runs=0,+ outcome_risk_runs=1,+ )++ response = client.get("/v1/workflows/wpid_target/reliability")++ assert response.status_code == 200+ body = response.json()+ assert body == {+ "state": "watch",+ "outcome_risk": True,+ "scored": True,+ "window_runs": 20,+ "healed_runs": 2,+ "heal_rate": 0.1,+ "consecutive_healed_runs": 1,+ "floor_runs": 0,+ "outcome_risk_runs": 1,+ }+ get_reliability.assert_awaited_once_with(ORG_ID, "wpid_target")