Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

2.1 KB
Raw Blame
from tqdm import tqdm
import torch
import pandas as pd

# DEVICE_TYPE can be "hip"/"mlx", which .to() rejects.
from unsloth.device_type import DEVICE_TYPE_TORCH

model_comparison_results = {}


def ppl_model(model, tokenizer, dataset):
    nlls = []
    max_length = 2048
    stride = 512
    for s in tqdm(range(len(dataset["text"]))):
        encodings = tokenizer(dataset["text"][s], return_tensors = "pt")
        seq_len = encodings.input_ids.size(1)
        prev_end_loc = 0
        for begin_loc in range(0, seq_len, stride):
            end_loc = min(begin_loc + max_length, seq_len)
            trg_len = end_loc - prev_end_loc
            input_ids = encodings.input_ids[:, begin_loc:end_loc].to(DEVICE_TYPE_TORCH)
            target_ids = input_ids.clone()
            target_ids[:, :-trg_len] = -100
            pad_token_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else 0
            attention_mask = (input_ids != pad_token_id).long()
            with torch.no_grad():
                outputs = model(input_ids, labels = target_ids, attention_mask = attention_mask)
                neg_log_likelihood = outputs.loss
            nlls.append(neg_log_likelihood)
            prev_end_loc = end_loc
            if end_loc == seq_len:
                break
    ppl = torch.exp(torch.stack(nlls).mean())
    return ppl


def add_to_comparison(model_name, ppl):
    """Record a model's perplexity in the comparison tracker."""
    model_comparison_results[model_name] = {"ppl": ppl}


def print_model_comparison():
    """Print a comparison of all models evaluated so far"""
    if not model_comparison_results:
        print("No model results available for comparison")
        return

    print("\n==== MODEL COMPARISON REPORT ====")

    comparison_df = pd.DataFrame(
        {
            "Model": list(model_comparison_results.keys()),
            "Perplexity": [
                results["ppl"].cpu().item() if torch.is_tensor(results["ppl"]) else results["ppl"]
                for results in model_comparison_results.values()
            ],
        }
    )

    print("\nComparison Table:")
    print(comparison_df.to_string(index = False))