Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

fix benchmark actions (#508)

Artur Mullakhmetovd4b077fc40f5
diff --git a/.github/workflows/run-benchmark.yml b/.github/workflows/run-benchmark.yml
index 5ff03ae24..40b1638c8 100644
--- a/.github/workflows/run-benchmark.yml
+++ b/.github/workflows/run-benchmark.yml
@@ -12,12 +12,20 @@ on:
- macos-m1-pool
- macos-m2-pool
- macos-m2-pro-pool
+ - macos-m3-max-pool
- macos-m4-pool
- macos-m4-pro-pool
+ - macos-m4-max-pool
+
benchmark_args:
description: 'JSON array passed after `uv run benchmarks`'
required: true
type: string
+ upload_metrics:
+ description: "Upload benchmark results to the metrics API"
+ required: false
+ default: false
+ type: boolean
permissions:
contents: read
@@ -39,6 +47,7 @@ jobs:
GITHUB_TRIGGERING_ACTOR_NAME: ${{ github.triggering_actor }}
POOL_NAME: ${{ inputs.pool }}
PREFECT_API_URL: ${{ vars.PREFECT_API_URL }}
+ UPLOAD_METRICS: ${{ inputs.upload_metrics }}
steps:
- uses: actions/checkout@v6
@@ -52,13 +61,18 @@ jobs:
PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
run: |
ARTIFACT_DIR="$RUNNER_TEMP/prefect-benchmark"
+ UPLOAD_METRICS_ARGS=()
+ if [[ "$UPLOAD_METRICS" == "true" ]]; then
+ UPLOAD_METRICS_ARGS+=(--upload-metrics)
+ fi
python3 scripts/prefect_benchmark_run.py \
--api-url "$PREFECT_API_URL" \
--pool "$POOL_NAME" \
--benchmark-args "$BENCHMARK_ARGS_JSON" \
--output-dir "$ARTIFACT_DIR" \
--actor "$GITHUB_ACTOR_NAME" \
- --triggering-actor "$GITHUB_TRIGGERING_ACTOR_NAME"
+ --triggering-actor "$GITHUB_TRIGGERING_ACTOR_NAME" \
+ "${UPLOAD_METRICS_ARGS[@]}"
- name: Upload Prefect benchmark artifacts
if: ${{ always() }}
diff --git a/.github/workflows/run-uzu-cargo-command.yml b/.github/workflows/run-uzu-cargo-command.yml
index cd1dd1b8c..28a879288 100644
--- a/.github/workflows/run-uzu-cargo-command.yml
+++ b/.github/workflows/run-uzu-cargo-command.yml
@@ -12,8 +12,10 @@ on:
- macos-m1-pool
- macos-m2-pool
- macos-m2-pro-pool
+ - macos-m3-max-pool
- macos-m4-pool
- macos-m4-pro-pool
+ - macos-m4-max-pool
cargo_args:
description: 'JSON array passed after `cargo`'
required: true
diff --git a/scripts/prefect_benchmark_run.py b/scripts/prefect_benchmark_run.py
index 74ac718d1..b5d5184a6 100644
--- a/scripts/prefect_benchmark_run.py
+++ b/scripts/prefect_benchmark_run.py
@@ -29,8 +29,10 @@ POOL_DEPLOYMENTS: dict[PoolName, str] = {
PoolName.MACOS_M1: "run-benchmark-worker-flow/run-benchmark-m1",
PoolName.MACOS_M2: "run-benchmark-worker-flow/run-benchmark-m2",
PoolName.MACOS_M2_PRO: "run-benchmark-worker-flow/run-benchmark-m2-pro",
+ PoolName.MACOS_M3_MAX: "run-benchmark-worker-flow/run-benchmark-m3-max",
PoolName.MACOS_M4: "run-benchmark-worker-flow/run-benchmark-m4",
PoolName.MACOS_M4_PRO: "run-benchmark-worker-flow/run-benchmark-m4-pro",
+ PoolName.MACOS_M4_MAX: "run-benchmark-worker-flow/run-benchmark-m4-max",
}
@@ -44,6 +46,7 @@ class BenchmarkRunResult:
benchmark_args: tuple[str, ...] = ()
actor: str | None = None
triggering_actor: str | None = None
+ upload_metrics: bool = False
@property
def succeeded(self) -> bool:
@@ -56,6 +59,7 @@ class CliConfig:
api_key: str
pool: PoolName
benchmark_args: tuple[str, ...]
+ upload_metrics: bool
output_dir: Path
poll_interval_seconds: int
timeout_seconds: int
@@ -71,6 +75,7 @@ class PrefectClientProtocol(Protocol):
self,
deployment_id: str,
benchmark_args: tuple[str, ...],
+ upload_metrics: bool = False,
) -> FlowRun:
...
@@ -86,19 +91,42 @@ class PrefectApiClient(BasePrefectApiClient):
self,
deployment_id: str,
benchmark_args: tuple[str, ...],
+ upload_metrics: bool = False,
) -> FlowRun:
+ parameters: dict[str, str | bool] = parse_benchmark_run_parameters(benchmark_args)
+ parameters["upload_metrics"] = upload_metrics
response = self._request_json(
method="POST",
path=f"/deployments/{urllib.parse.quote(deployment_id, safe='')}/create_flow_run",
body={
- "parameters": {
- "benchmark_args": list(benchmark_args),
- }
+ "parameters": parameters,
},
)
return parse_flow_run(response, self.ui_base_url)
+def parse_benchmark_run_parameters(benchmark_args: tuple[str, ...]) -> dict[str, str]:
+ parser = argparse.ArgumentParser(
+ prog="uv run benchmarks",
+ description="Parse benchmark CLI arguments into Prefect worker flow parameters.",
+ allow_abbrev=False,
+ )
+ subparsers = parser.add_subparsers(dest="command", required=True)
+ run_parser = subparsers.add_parser("run", allow_abbrev=False)
+ run_parser.add_argument("identifier")
+ run_parser.add_argument("--task", required=True)
+ run_parser.add_argument("--profile", required=True)
+ run_parser.add_argument("--uzu-branch", "--uzu-ref", dest="uzu_ref", default="main")
+
+ parsed_args = parser.parse_args(list(benchmark_args))
+ return {
+ "identifier": parsed_args.identifier,
+ "task": parsed_args.task,
+ "profile": parsed_args.profile,
+ "uzu_ref": parsed_args.uzu_ref,
+ }
+
+
def parse_benchmark_args_json(raw_value: str) -> tuple[str, ...]:
parsed_value = json.loads(raw_value)
if not isinstance(parsed_value, list):
@@ -114,6 +142,7 @@ def execute_benchmark_run(
client: PrefectClientProtocol,
pool: PoolName,
benchmark_args: tuple[str, ...],
+ upload_metrics: bool,
ui_base_url: str,
poll_interval_seconds: int,
timeout_seconds: int,
@@ -124,7 +153,11 @@ def execute_benchmark_run(
) -> BenchmarkRunResult:
deployment_name = deployment_name_for_pool(pool)
deployment_id = client.read_deployment_id(deployment_name)
- created_run = client.create_flow_run_from_deployment(deployment_id, benchmark_args)
+ created_run = client.create_flow_run_from_deployment(
+ deployment_id,
+ benchmark_args,
+ upload_metrics=upload_metrics,
+ )
observed_run = observe_flow_run(
client=client,
created_run=created_run,
@@ -142,6 +175,7 @@ def execute_benchmark_run(
logs=observed_run.logs,
failure_reason=observed_run.failure_reason,
benchmark_args=benchmark_args,
+ upload_metrics=upload_metrics,
actor=actor,
triggering_actor=triggering_actor,
)
@@ -171,6 +205,7 @@ def build_summary_payload(result: BenchmarkRunResult) -> dict[str, Any]:
"pool": result.pool.value,
"succeeded": result.succeeded,
"triggering_actor": result.triggering_actor,
+ "upload_metrics": result.upload_metrics,
}
@@ -187,6 +222,7 @@ def build_summary_markdown(result: BenchmarkRunResult) -> str:
f"- actor: `{result.actor or 'unknown'}`",
f"- triggering actor: `{result.triggering_actor or result.actor or 'unknown'}`",
f"- benchmark args: `{json.dumps(list(result.benchmark_args))}`",
+ f"- upload metrics: `{'true' if result.upload_metrics else 'false'}`",
]
if result.failure_reason is not None:
lines.append(f"- failure reason: `{result.failure_reason}`")
@@ -199,7 +235,7 @@ def parse_cli_args(argv: list[str] | None = None) -> CliConfig:
parser.add_argument(
"--pool",
required=True,
- choices=[pool.value for pool in PoolName],
+ choices=[pool.value for pool in POOL_DEPLOYMENTS],
help="Benchmark worker pool to target.",
)
parser.add_argument(
@@ -208,6 +244,12 @@ def parse_cli_args(argv: list[str] | None = None) -> CliConfig:
help='JSON array of argv passed after "uv run benchmarks".',
)
parser.add_argument("--output-dir", required=True, help="Directory for summary and log artifacts.")
+ parser.add_argument(
+ "--upload-metrics",
+ action="store_true",
+ default=False,
+ help="Upload benchmark results to the metrics API after the benchmark completes.",
+ )
parser.add_argument("--poll-interval-seconds", type=int, default=15, help="Polling interval for flow run state.")
parser.add_argument("--timeout-seconds", type=int, default=21600, help="Overall timeout for waiting on the flow run.")
parser.add_argument("--actor", default=None, help="GitHub actor that started the workflow.")
@@ -224,6 +266,7 @@ def parse_cli_args(argv: list[str] | None = None) -> CliConfig:
api_key=api_key,
pool=PoolName(parsed_args.pool),
benchmark_args=parse_benchmark_args_json(parsed_args.benchmark_args),
+ upload_metrics=parsed_args.upload_metrics,
output_dir=Path(parsed_args.output_dir),
poll_interval_seconds=parsed_args.poll_interval_seconds,
timeout_seconds=parsed_args.timeout_seconds,
@@ -243,6 +286,7 @@ def main(argv: list[str] | None = None) -> int:
client=client,
pool=config.pool,
benchmark_args=config.benchmark_args,
+ upload_metrics=config.upload_metrics,
ui_base_url=client.ui_base_url,
poll_interval_seconds=config.poll_interval_seconds,
timeout_seconds=config.timeout_seconds,
diff --git a/scripts/prefect_flow_run_common.py b/scripts/prefect_flow_run_common.py
index 61d608897..5b2655e6b 100644
--- a/scripts/prefect_flow_run_common.py
+++ b/scripts/prefect_flow_run_common.py
@@ -20,8 +20,10 @@ class PoolName(str, Enum):
MACOS_M1 = "macos-m1-pool"
MACOS_M2 = "macos-m2-pool"
MACOS_M2_PRO = "macos-m2-pro-pool"
+ MACOS_M3_MAX = "macos-m3-max-pool"
MACOS_M4 = "macos-m4-pool"
MACOS_M4_PRO = "macos-m4-pro-pool"
+ MACOS_M4_MAX = "macos-m4-max-pool"
class FlowRunStateType(str, Enum):
diff --git a/scripts/prefect_uzu_cargo_run.py b/scripts/prefect_uzu_cargo_run.py
index 526803884..dd28510a0 100644
--- a/scripts/prefect_uzu_cargo_run.py
+++ b/scripts/prefect_uzu_cargo_run.py
@@ -29,8 +29,10 @@ POOL_DEPLOYMENTS: dict[PoolName, str] = {
PoolName.MACOS_M1: "run-uzu-cargo-command-worker-flow/run-uzu-cargo-command-m1",
PoolName.MACOS_M2: "run-uzu-cargo-command-worker-flow/run-uzu-cargo-command-m2",
PoolName.MACOS_M2_PRO: "run-uzu-cargo-command-worker-flow/run-uzu-cargo-command-m2-pro",
+ PoolName.MACOS_M3_MAX: "run-uzu-cargo-command-worker-flow/run-uzu-cargo-command-m3-max",
PoolName.MACOS_M4: "run-uzu-cargo-command-worker-flow/run-uzu-cargo-command-m4",
PoolName.MACOS_M4_PRO: "run-uzu-cargo-command-worker-flow/run-uzu-cargo-command-m4-pro",
+ PoolName.MACOS_M4_MAX: "run-uzu-cargo-command-worker-flow/run-uzu-cargo-command-m4-max",
}
@@ -212,7 +214,7 @@ def parse_cli_args(argv: list[str] | None = None) -> CliConfig:
parser.add_argument(
"--pool",
required=True,
- choices=[pool.value for pool in PoolName],
+ choices=[pool.value for pool in POOL_DEPLOYMENTS],
help="MacOS pool to target.",
)
parser.add_argument(