Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

1.1 KB
Raw Blame
#ifndef __engine_llamacpp_engine_hpp__
#define __engine_llamacpp_engine_hpp__

#include <chat.h>
#include <llama-cpp.h>

#include <string>
#include <vector>

#include "common.hpp"

class LlamaEngine final : public InferenceEngine {
public:
    explicit LlamaEngine(
        const std::string& model,
        const std::string& draft_model = ""
    );

    std::vector<BenchResponse> execute(const BenchRequest& request) const override;

private:
    struct RunConfig;

    std::vector<llama_token> tokenize(const BenchRequest& request) const;

    BenchResponse run_single(
        std::vector<llama_token> tokens,
        const RunConfig& config,
        const llama_sampler_ptr& sampler_chain
    ) const;

    llama_model_ptr model;
    llama_model_ptr draft_model;
    size_t dflash_depth = 0;
    const llama_vocab* tokenizer;
    common_chat_templates_ptr chat_templates;
    bool has_mtp;
};

std::string decode_token(
    const llama_vocab* vocab,
    llama_token token
);

bool has_mtp_weights(const std::filesystem::path& model_path);

#endif  // __engine_llamacpp_engine_hpp__