Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

1.6 KB
Raw Blame
from pydantic import BaseModel, Field, JsonValue


class ChatMessage(BaseModel):
    role: str
    content: str | None = None
    reasoning_content: str | None = None
    tool_calls: list[JsonValue] | None = Field(default=None, exclude_if=lambda value: value is None)
    tool_call_id: str | None = Field(default=None, exclude_if=lambda value: value is None)


class BenchSampling(BaseModel):
    top_k: int | None = None
    top_p: float | None = None
    min_p: float | None = None
    temp: float | None = None


class BenchRequest(BaseModel):
    prompt_text: str | None = None
    prompt_chat: list[ChatMessage] | None = None
    tools: list[JsonValue] | None = Field(default=None, exclude_if=lambda value: value is None)
    tool_choice: JsonValue | None = Field(default=None, exclude_if=lambda value: value is None)

    max_tokens: int | None = Field(default=None, ge=0)
    speculative_depth: int | None = Field(default=None, ge=0)
    sampling: BenchSampling | None = None
    num_runs: int | None = Field(default=None, ge=0)

    @property
    def prompt(self) -> str | list[ChatMessage]:
        if self.prompt_text is not None:
            return self.prompt_text
        elif self.prompt_chat is not None:
            return self.prompt_chat
        else:
            raise ValueError("prompt_text and prompt_chat are None")


class BenchResponse(BaseModel):
    text: str
    tokens_count: int
    time_to_first_token: float
    prompt_tps: float
    decode_tps: float
    tokens_per_forward_pass: float
    duration: float
    memory_phys_footprint: int
    memory_resident: int
    memory_graphics_total: int