Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

10.4 KB
Raw Blame
from __future__ import annotations

import argparse
import json
import os
import time
import urllib.parse
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Protocol

from prefect_flow_run_common import (
    FlowRun,
    FlowRunStateType,
    LogSink,
    PoolName,
    PrefectApiClient as BasePrefectApiClient,
    PrefectLog,
    observe_flow_run,
    parse_flow_run,
    print_live_log_line,
    ui_base_url_from_api_url,
    write_artifacts,
)


POOL_DEPLOYMENTS: dict[PoolName, str] = {
    PoolName.MACOS_A18_PRO: "run-benchmark-worker-flow/run-benchmark-a18-pro",
    PoolName.MACOS_M1: "run-benchmark-worker-flow/run-benchmark-m1",
    PoolName.MACOS_M2: "run-benchmark-worker-flow/run-benchmark-m2",
    PoolName.MACOS_M2_PRO: "run-benchmark-worker-flow/run-benchmark-m2-pro",
    PoolName.MACOS_M3_MAX: "run-benchmark-worker-flow/run-benchmark-m3-max",
    PoolName.MACOS_M4: "run-benchmark-worker-flow/run-benchmark-m4",
    PoolName.MACOS_M4_PRO: "run-benchmark-worker-flow/run-benchmark-m4-pro",
    PoolName.MACOS_M4_MAX: "run-benchmark-worker-flow/run-benchmark-m4-max",
}


@dataclass(frozen=True)
class BenchmarkRunResult:
    deployment_name: str
    pool: PoolName
    flow_run: FlowRun
    logs: tuple[PrefectLog, ...]
    failure_reason: str | None = None
    benchmark_args: tuple[str, ...] = ()
    actor: str | None = None
    triggering_actor: str | None = None
    upload_metrics: bool = False

    @property
    def succeeded(self) -> bool:
        return self.failure_reason is None and self.flow_run.state_type == FlowRunStateType.COMPLETED


@dataclass(frozen=True)
class CliConfig:
    api_url: str
    api_key: str
    pool: PoolName
    benchmark_args: tuple[str, ...]
    upload_metrics: bool
    output_dir: Path
    poll_interval_seconds: int
    timeout_seconds: int
    actor: str | None
    triggering_actor: str | None


class PrefectClientProtocol(Protocol):
    def read_deployment_id(self, deployment_name: str) -> str:
        ...

    def create_flow_run_from_deployment(
        self,
        deployment_id: str,
        benchmark_args: tuple[str, ...],
        upload_metrics: bool = False,
    ) -> FlowRun:
        ...

    def read_flow_run(self, flow_run_id: str) -> FlowRun:
        ...

    def read_logs_page(self, flow_run_id: str, *, limit: int, offset: int) -> tuple[PrefectLog, ...]:
        ...


class PrefectApiClient(BasePrefectApiClient):
    def create_flow_run_from_deployment(
        self,
        deployment_id: str,
        benchmark_args: tuple[str, ...],
        upload_metrics: bool = False,
    ) -> FlowRun:
        parameters: dict[str, str | bool] = parse_benchmark_run_parameters(benchmark_args)
        parameters["upload_metrics"] = upload_metrics
        response = self._request_json(
            method="POST",
            path=f"/deployments/{urllib.parse.quote(deployment_id, safe='')}/create_flow_run",
            body={
                "parameters": parameters,
            },
        )
        return parse_flow_run(response, self.ui_base_url)


def parse_benchmark_run_parameters(benchmark_args: tuple[str, ...]) -> dict[str, str]:
    parser = argparse.ArgumentParser(
        prog="uv run benchmarks",
        description="Parse benchmark CLI arguments into Prefect worker flow parameters.",
        allow_abbrev=False,
    )
    subparsers = parser.add_subparsers(dest="command", required=True)
    run_parser = subparsers.add_parser("run", allow_abbrev=False)
    run_parser.add_argument("identifier")
    run_parser.add_argument("--task", required=True)
    run_parser.add_argument("--profile", required=True)
    run_parser.add_argument("--uzu-branch", "--uzu-ref", dest="uzu_ref", default="main")

    parsed_args = parser.parse_args(list(benchmark_args))
    return {
        "identifier": parsed_args.identifier,
        "task": parsed_args.task,
        "profile": parsed_args.profile,
        "uzu_ref": parsed_args.uzu_ref,
    }


def parse_benchmark_args_json(raw_value: str) -> tuple[str, ...]:
    parsed_value = json.loads(raw_value)
    if not isinstance(parsed_value, list):
        raise ValueError("benchmark_args must decode to a JSON array")
    return tuple(str(item) for item in parsed_value)


def deployment_name_for_pool(pool: PoolName) -> str:
    return POOL_DEPLOYMENTS[pool]


def execute_benchmark_run(
    client: PrefectClientProtocol,
    pool: PoolName,
    benchmark_args: tuple[str, ...],
    upload_metrics: bool,
    ui_base_url: str,
    poll_interval_seconds: int,
    timeout_seconds: int,
    sleep: Callable[[float], None] = time.sleep,
    actor: str | None = None,
    triggering_actor: str | None = None,
    log_sink: LogSink | None = None,
) -> BenchmarkRunResult:
    deployment_name = deployment_name_for_pool(pool)
    deployment_id = client.read_deployment_id(deployment_name)
    created_run = client.create_flow_run_from_deployment(
        deployment_id,
        benchmark_args,
        upload_metrics=upload_metrics,
    )
    observed_run = observe_flow_run(
        client=client,
        created_run=created_run,
        deployment_name=deployment_name,
        ui_base_url=ui_base_url,
        poll_interval_seconds=poll_interval_seconds,
        timeout_seconds=timeout_seconds,
        sleep=sleep,
        log_sink=log_sink,
    )
    return BenchmarkRunResult(
        deployment_name=observed_run.deployment_name,
        pool=pool,
        flow_run=observed_run.flow_run,
        logs=observed_run.logs,
        failure_reason=observed_run.failure_reason,
        benchmark_args=benchmark_args,
        upload_metrics=upload_metrics,
        actor=actor,
        triggering_actor=triggering_actor,
    )


def write_run_artifacts(output_dir: Path | str, result: BenchmarkRunResult) -> None:
    write_artifacts(
        output_dir,
        summary_payload=build_summary_payload(result),
        summary_markdown=build_summary_markdown(result),
        flow_run=result.flow_run,
        logs=result.logs,
    )


def build_summary_payload(result: BenchmarkRunResult) -> dict[str, Any]:
    return {
        "actor": result.actor,
        "benchmark_args": list(result.benchmark_args),
        "deployment_name": result.deployment_name,
        "failure_reason": result.failure_reason,
        "flow_run_id": result.flow_run.id,
        "flow_run_name": result.flow_run.name,
        "flow_run_state_name": result.flow_run.state_name,
        "flow_run_state_type": result.flow_run.state_type.value,
        "flow_run_url": result.flow_run.url,
        "pool": result.pool.value,
        "succeeded": result.succeeded,
        "triggering_actor": result.triggering_actor,
        "upload_metrics": result.upload_metrics,
    }


def build_summary_markdown(result: BenchmarkRunResult) -> str:
    lines = [
        "## Prefect Benchmark Run",
        "",
        f"- status: `{'succeeded' if result.succeeded else 'failed'}`",
        f"- pool: `{result.pool.value}`",
        f"- deployment: `{result.deployment_name}`",
        f"- flow run: `{result.flow_run.name}` (`{result.flow_run.id}`)",
        f"- state: `{result.flow_run.state_name}`",
        f"- url: {result.flow_run.url}",
        f"- actor: `{result.actor or 'unknown'}`",
        f"- triggering actor: `{result.triggering_actor or result.actor or 'unknown'}`",
        f"- benchmark args: `{json.dumps(list(result.benchmark_args))}`",
        f"- upload metrics: `{'true' if result.upload_metrics else 'false'}`",
    ]
    if result.failure_reason is not None:
        lines.append(f"- failure reason: `{result.failure_reason}`")
    return "\n".join(lines) + "\n"


def parse_cli_args(argv: list[str] | None = None) -> CliConfig:
    parser = argparse.ArgumentParser(description="Trigger one Prefect benchmark worker deployment and fetch its logs.")
    parser.add_argument("--api-url", required=True, help="Prefect API URL")
    parser.add_argument(
        "--pool",
        required=True,
        choices=[pool.value for pool in POOL_DEPLOYMENTS],
        help="Benchmark worker pool to target.",
    )
    parser.add_argument(
        "--benchmark-args",
        required=True,
        help='JSON array of argv passed after "uv run benchmarks".',
    )
    parser.add_argument("--output-dir", required=True, help="Directory for summary and log artifacts.")
    parser.add_argument(
        "--upload-metrics",
        action="store_true",
        default=False,
        help="Upload benchmark results to the metrics API after the benchmark completes.",
    )
    parser.add_argument("--poll-interval-seconds", type=int, default=15, help="Polling interval for flow run state.")
    parser.add_argument("--timeout-seconds", type=int, default=21600, help="Overall timeout for waiting on the flow run.")
    parser.add_argument("--actor", default=None, help="GitHub actor that started the workflow.")
    parser.add_argument("--triggering-actor", default=None, help="GitHub actor that triggered this run or re-run.")
    parsed_args = parser.parse_args(argv)
    api_key = os.environ.get("PREFECT_API_KEY")
    if not api_key:
        raise SystemExit("PREFECT_API_KEY environment variable is required")
    api_url = parsed_args.api_url.strip()
    if not api_url:
        raise SystemExit("--api-url must be non-empty")
    return CliConfig(
        api_url=api_url,
        api_key=api_key,
        pool=PoolName(parsed_args.pool),
        benchmark_args=parse_benchmark_args_json(parsed_args.benchmark_args),
        upload_metrics=parsed_args.upload_metrics,
        output_dir=Path(parsed_args.output_dir),
        poll_interval_seconds=parsed_args.poll_interval_seconds,
        timeout_seconds=parsed_args.timeout_seconds,
        actor=parsed_args.actor,
        triggering_actor=parsed_args.triggering_actor,
    )


def main(argv: list[str] | None = None) -> int:
    config = parse_cli_args(argv)
    client = PrefectApiClient(
        api_url=config.api_url.rstrip("/"),
        api_key=config.api_key,
        ui_base_url=ui_base_url_from_api_url(config.api_url),
    )
    result = execute_benchmark_run(
        client=client,
        pool=config.pool,
        benchmark_args=config.benchmark_args,
        upload_metrics=config.upload_metrics,
        ui_base_url=client.ui_base_url,
        poll_interval_seconds=config.poll_interval_seconds,
        timeout_seconds=config.timeout_seconds,
        actor=config.actor,
        triggering_actor=config.triggering_actor,
        log_sink=print_live_log_line,
    )
    write_run_artifacts(config.output_dir, result)
    print(build_summary_markdown(result))
    return 0 if result.succeeded else 1


if __name__ == "__main__":
    raise SystemExit(main())