Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

1.5 KB
Raw Blame
#ifndef __benchmarks_bench_hpp__
#define __benchmarks_bench_hpp__

#include <cstddef>
#include <cstdint>
#include <glaze/json/generic.hpp>
#include <optional>
#include <string>
#include <vector>

struct ChatMessage {
    std::string role;
    std::optional<std::string> content = std::nullopt;
    std::optional<std::string> reasoning_content = std::nullopt;
    std::optional<std::vector<glz::generic_u64>> tool_calls = std::nullopt;
    std::optional<std::string> tool_call_id = std::nullopt;
};

struct BenchSampling {
    std::optional<int32_t> top_k = std::nullopt;
    std::optional<float> top_p = std::nullopt;
    std::optional<float> min_p = std::nullopt;
    std::optional<float> temp = std::nullopt;
};

struct BenchRequest {
    std::optional<std::string> prompt_text = std::nullopt;
    std::optional<std::vector<ChatMessage>> prompt_chat = std::nullopt;
    std::optional<std::vector<glz::generic_u64>> tools = std::nullopt;
    std::optional<glz::generic_u64> tool_choice = std::nullopt;

    std::optional<size_t> max_tokens = std::nullopt;
    std::optional<size_t> speculative_depth = std::nullopt;
    std::optional<BenchSampling> sampling = std::nullopt;
    std::optional<size_t> num_runs = std::nullopt;
};

struct BenchResponse {
    std::string text;
    size_t tokens_count;
    double time_to_first_token;
    double prompt_tps;
    double decode_tps;
    double tokens_per_forward_pass;
    double duration;
    uint64_t memory_phys_footprint;
    uint64_t memory_resident;
    uint64_t memory_graphics_total;
};

#endif  // __benchmarks_bench_hpp__