Skip to content

Self-improving AI Employees for everyone

10.6 KB
Raw Blame
"""Real Temporal admission/Reset protocol against a native Temporal CLI.

Runs only when TEMPORAL_TEST_CLI names a native Temporal CLI binary. The test
never looks for one itself: a Temporal install on PATH or in the npm global
prefix belongs to the developer, not to a test run. It never downloads a
server or connects to the application's running Temporal server either.
"""

from __future__ import annotations

import asyncio
import os
import subprocess
import sys
from contextlib import suppress
from datetime import timedelta
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4

import pytest


def _existing_cli() -> Path | None:
    configured = os.environ.get("TEMPORAL_TEST_CLI")
    if not configured:
        return None
    path = Path(configured)
    return path if path.is_file() else None


async def _run_gate(cli: str) -> None:
    from temporalio import activity
    from temporalio.api.enums.v1 import EventType
    from temporalio.client import (
        WithStartWorkflowOperation,
        WorkflowExecutionStatus,
        WorkflowUpdateFailedError,
    )
    from temporalio.common import WorkflowIDConflictPolicy
    from temporalio.testing import WorkflowEnvironment
    from temporalio.worker import Replayer, Worker

    from services.temporal.node_invocation import NodeInvocationWorkflow
    from services.temporal.workspace_tasks_workflow import WorkspaceTaskControllerWorkflow

    token = uuid4().hex
    task_queue = f"workspace-reset-test-{token}"
    controller_id = f"workspace-controller-{token}"
    workflow_id = f"workspace-graph-{token}"
    first_id, second_id = f"device-first-{token}", f"device-second-{token}"
    devices = {
        run_id: SimpleNamespace(
            started=asyncio.Event(), finish=asyncio.Event(), cancelled=asyncio.Event(),
            release_cleanup=asyncio.Event(), cleaned=asyncio.Event(),
        )
        for run_id in (first_id, second_id)
    }

    class FakeActivities:
        def __init__(self):
            self.cleanup_started = asyncio.Event()
            self.release_cleanup = asyncio.Event()
            self.cleanup_calls = 0
            self.summaries = set()

        @activity.defn(name="test.workspace.device")
        async def device(self, context: dict) -> dict:
            state = devices[context["execution_id"]]
            state.started.set()
            try:
                while not state.finish.is_set():
                    activity.heartbeat("fake device is running")
                    try:
                        await asyncio.wait_for(state.finish.wait(), 0.05)
                    except TimeoutError:
                        pass
                return {"success": True}
            except asyncio.CancelledError:
                state.cancelled.set()
                await state.release_cleanup.wait()
                state.cleaned.set()
                raise

        @activity.defn(name="workflow_runs.record_completion")
        async def record(self, payload: dict) -> dict:
            self.summaries.add(payload["run_id"].split(":", 1)[0])
            return {"recorded": True}

        @activity.defn(name="workspace_tasks.reset_runtime")
        async def reset_runtime(self, payload: dict) -> dict:
            assert payload == {"workflow_id": workflow_id}
            assert devices[first_id].cleaned.is_set()
            assert first_id in self.summaries
            self.cleanup_calls += 1
            self.cleanup_started.set()
            while not self.release_cleanup.is_set():
                activity.heartbeat("fake plugin cleanup is running")
                try:
                    await asyncio.wait_for(self.release_cleanup.wait(), 0.05)
                except TimeoutError:
                    pass
            return {"reset_nodes": ["phone-one"]}

    activities = FakeActivities()

    def submission(run_id: str, epoch: int) -> dict:
        return {
            "workflow_id": workflow_id, "node_id": "phone-one", "principal": "test-owner",
            "fingerprint": run_id, "admission_epoch": epoch,
            "activity": "test.workspace.device", "task_queue": task_queue, "timeout_s": 60,
            "context": {"workflow_id": workflow_id, "execution_id": run_id},
        }

    # An explicit native binary disables SDK downloads. The omitted database
    # filename selects in-memory SQLite, and port=None selects a free port.
    async with await WorkflowEnvironment.start_local(
        dev_server_existing_path=cli, ip="127.0.0.1", port=None,
        ui=False, dev_server_log_level="error",
    ) as environment:
        client = environment.client

        async def update(method, argument):
            return await client.execute_update_with_start_workflow(
                method, argument, id=uuid4().hex,
                start_workflow_operation=WithStartWorkflowOperation(
                    WorkspaceTaskControllerWorkflow.run, {"workflow_id": workflow_id},
                    id=controller_id, task_queue=task_queue,
                    id_conflict_policy=WorkflowIDConflictPolicy.USE_EXISTING,
                ),
            )

        async def refused(request, error_type):
            with pytest.raises(WorkflowUpdateFailedError) as rejected:
                await update(WorkspaceTaskControllerWorkflow.submit, request)
            assert rejected.value.cause.type == error_type

        async with Worker(
            client, task_queue=task_queue,
            workflows=[WorkspaceTaskControllerWorkflow, NodeInvocationWorkflow],
            activities=[activities.device, activities.record, activities.reset_runtime],
            max_heartbeat_throttle_interval=timedelta(milliseconds=50),
            default_heartbeat_throttle_interval=timedelta(milliseconds=50),
        ):
            controller = client.get_workflow_handle(controller_id)
            reset_task = None
            try:
                assert await update(WorkspaceTaskControllerWorkflow.submit, submission(first_id, 0)) == {
                    "run_id": first_id, "status": "accepted",
                }
                await asyncio.wait_for(devices[first_id].started.wait(), 10)
                assert (await controller.query(WorkspaceTaskControllerWorkflow.describe))["active_count"] == 1

                reset_task = asyncio.create_task(update(WorkspaceTaskControllerWorkflow.reset, "reset-one"))
                await asyncio.wait_for(devices[first_id].cancelled.wait(), 10)
                assert not reset_task.done()
                assert not activities.cleanup_started.is_set()
                assert (await controller.query(WorkspaceTaskControllerWorkflow.describe))["reset_request_id"] == "reset-one"
                await refused(submission(second_id, 1), "WorkspaceResetInProgress")

                devices[first_id].release_cleanup.set()
                await asyncio.wait_for(activities.cleanup_started.wait(), 10)
                assert not reset_task.done()
                state = await controller.query(WorkspaceTaskControllerWorkflow.describe)
                assert state["active_count"] == 0 and state["resetting"] and state["epoch"] == 1
                assert (await client.get_workflow_handle(first_id).describe()).status == WorkflowExecutionStatus.CANCELED
                await refused(submission(second_id, 1), "WorkspaceResetInProgress")

                activities.release_cleanup.set()
                reset_result = {"cancelled": 1, "reset_nodes": ["phone-one"]}
                assert await asyncio.wait_for(reset_task, 10) == reset_result
                await refused(submission(second_id, 0), "WorkspaceAdmissionChanged")
                assert (await update(WorkspaceTaskControllerWorkflow.submit, submission(second_id, 1)))["status"] == "accepted"
                await asyncio.wait_for(devices[second_id].started.wait(), 10)
                # Fresh RPC update ID, same logical reset: never cancel work
                # admitted after that reset or rerun runtime cleanup.
                assert await update(WorkspaceTaskControllerWorkflow.reset, "reset-one") == reset_result
                assert not devices[second_id].cancelled.is_set()
                assert activities.cleanup_calls == 1
                devices[second_id].finish.set()
                assert await asyncio.wait_for(client.get_workflow_handle(second_id).result(), 10) == {"success": True}

                async def registry_empty():
                    while (await controller.query(WorkspaceTaskControllerWorkflow.describe))["active_count"]:
                        await asyncio.sleep(0.05)

                await asyncio.wait_for(registry_empty(), 10)
                history = await controller.fetch_history()
                child_closed = next(event.event_id for event in history.events if
                    event.event_type == EventType.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_CANCELED)
                cleanup_scheduled = next(event.event_id for event in history.events if
                    event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED
                    and event.activity_task_scheduled_event_attributes.activity_type.name == "workspace_tasks.reset_runtime")
                assert child_closed < cleanup_scheduled
                replay = await Replayer(workflows=[WorkspaceTaskControllerWorkflow]).replay_workflow(history)
                assert replay.replay_failure is None
            finally:
                for device in devices.values():
                    device.finish.set()
                    device.release_cleanup.set()
                activities.release_cleanup.set()
                # Only this in-memory test controller is terminated. Release
                # every fake cleanup gate even after a failed assertion so
                # Worker/server context managers can always stop promptly.
                with suppress(Exception):
                    await controller.terminate("isolated integration test finished")
                if reset_task is not None:
                    with suppress(TimeoutError):
                        await asyncio.wait_for(asyncio.gather(reset_task, return_exceptions=True), 5)


def test_real_temporal_admission_reset_waits_for_child_and_plugin_cleanup():
    cli = _existing_cli()
    if cli is None:
        pytest.skip("Set TEMPORAL_TEST_CLI to a native Temporal CLI binary to run this test")
    completed = subprocess.run(
        [sys.executable, "-X", "utf8", "-m", "tests.temporal.test_workspace_tasks_integration", str(cli)],
        cwd=Path(__file__).parents[2], stdin=subprocess.DEVNULL,
        capture_output=True, text=True, encoding="utf-8", timeout=120, check=False,
    )
    assert completed.returncode == 0, completed.stdout + completed.stderr


if __name__ == "__main__":
    asyncio.run(_run_gate(sys.argv[1]))