Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

1.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

from __future__ import annotations

import math
from typing import Any, Optional


EVAL_SPLIT_CANDIDATES = ("eval", "validation", "valid", "val", "test")
MIN_EVAL_ROWS = 16
MIN_TOTAL_ROWS_FOR_EVAL = MIN_EVAL_ROWS * 2
# Streaming eval has no __len__, so cap it or an unbounded eval iterates the whole source on every eval step.
STREAMING_EVAL_MAX_SAMPLES = 500


def evaluation_enabled(value: Any) -> bool:
    """Return whether a configured eval interval is finite and positive."""
    if isinstance(value, bool):
        return False
    try:
        interval = float(value)
    except (TypeError, ValueError, OverflowError):
        # OverflowError, not ValueError: float() refuses a JSON int too large to represent.
        return False
    return math.isfinite(interval) and interval > 0


def split_dataset_for_evaluation(dataset: Any, *, seed: int = 3407) -> Optional[tuple[Any, Any]]:
    """Create the bounded deterministic train/eval split used by both training backends."""
    total_rows = len(dataset)
    if total_rows < MIN_TOTAL_ROWS_FOR_EVAL:
        return None

    eval_rows = max(MIN_EVAL_ROWS, min(128, int(0.05 * total_rows)))
    eval_rows = min(eval_rows, total_rows // 2)
    if isinstance(dataset, list):
        import numpy as np
        order = np.random.default_rng(seed).permutation(total_rows)
        return [dataset[i] for i in order[eval_rows:]], [dataset[i] for i in order[:eval_rows]]
    split = dataset.train_test_split(test_size = eval_rows, seed = seed)
    return split["train"], split["test"]