ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
#!/usr/bin/env python3
"""Inspect a local GGUF/runtime and emit a portable, explicit MTP launch profile.
This command never downloads, starts, stops or switches an inference server.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import platform
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "extensions/services/dashboard-api"))
from gguf_inspector import inspect_gguf
from model_mtp import mtp_metadata, probe_runtime, recommend_mtp, validate_runtime_command
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for block in iter(lambda: source.read(8 * 1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--runtime", type=Path, required=True)
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--hardware-id", required=True, help="Exact accelerator/CPU identifier used for benchmark matching")
parser.add_argument("--context", type=int, default=16384)
parser.add_argument("--draft-tokens", type=int, choices=range(1, 7), default=2)
# llama-server launch only (the native argv ODS starts).
parser.add_argument("--launch-mode", choices=("native",), default="native")
parser.add_argument("--vision-projector", type=Path)
parser.add_argument("--expected-sha256", required=True)
parser.add_argument("--benchmark-evidence", type=Path)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
model, runtime = args.model.resolve(strict=True), args.runtime.resolve(strict=True)
if model.suffix.lower() != ".gguf" or not model.is_file() or args.context < 512:
parser.error("a completed GGUF and context >= 512 are required")
digest = sha256(model)
if digest != args.expected_sha256.lower():
parser.error("model SHA-256 does not match the publisher artifact")
inspection = inspect_gguf(model, max_metadata_bytes=32 * 1024 * 1024)
support = mtp_metadata({}, inspection)
if not inspection.get("readable") or not support or support["modelSupport"] != "embedded":
parser.error("this artifact does not declare embedded MTP; external draft models need separate qualification")
if args.context > (inspection.get("context_length") or 0):
parser.error("requested context exceeds the artifact's declared native context")
capability = probe_runtime(runtime)
load_args = capability.get("loadModeArguments", [])
projector = args.vision_projector.resolve(strict=True) if args.vision_projector else None
gpu_layers = "auto"
baseline = [str(runtime), "--model", str(model), "--jinja", "--ctx-size", str(args.context),
"--parallel", "1", "--gpu-layers", gpu_layers, "--flash-attn", "on",
"--cache-type-k", "q4_0", "--cache-type-v", "q4_0"]
if projector:
baseline += ["--mmproj", str(projector)]
if load_args:
baseline += load_args
mtp = baseline + ["--spec-type", "draft-mtp", "--spec-draft-n-max", str(args.draft_tokens),
"--spec-draft-type-k", "q4_0", "--spec-draft-type-v", "q4_0"]
for command in (baseline, mtp):
validate_runtime_command(command)
signature_data = {"modelSha256": digest, "runtimeSha256": sha256(runtime),
"hardware": args.hardware_id, "platform": platform.platform(),
"context": args.context, "draftTokens": args.draft_tokens,
"cacheType": "q4_0", "parallel": 1, "gpuLayers": gpu_layers, "flashAttention": "on",
"loadModeArguments":load_args}
signature = hashlib.sha256(json.dumps(signature_data, sort_keys=True).encode()).hexdigest()
evidence = json.loads(args.benchmark_evidence.read_text()) if args.benchmark_evidence else None
recommendation = recommend_mtp(evidence, signature) if capability["mtp"] else {"recommendation":"baseline","speedup":None}
result = {"schemaVersion":1, "signature":signature, "profile":signature_data,
"model":support, "runtime":capability, **recommendation,
"baselineCommand":baseline, "mtpCommand":mtp if capability["mtp"] else None,
"notes":["Launch arguments were parsed with --help without loading the model. Validate memory fit before serving traffic.",
"MTP is recommended only after at least three valid paired samples of 128+ tokens and a measured gain of 5%."]}
rendered = json.dumps(result, indent=2)
if args.output:
args.output.write_text(rendered + "\n", encoding="utf-8")
print(rendered)
return 0 if capability["mtp"] else 2
if __name__ == "__main__":
raise SystemExit(main())