Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

3.9 KB
Raw Blame
"""Mode-collapse probe (v0.56.0).

Diversity collapse signal: generate K completions per prompt at
temperature=0 and temperature=1, then score 1 - self-BLEU-style overlap
(actually averaged pairwise n-gram Jaccard distance — pure-python, no
nltk).
"""

from __future__ import annotations

import unicodedata
from typing import Callable, Sequence

from soup_cli.utils.diagnose._common import (
    jaccard,
    merge_evidence,
    ngrams,
    reject_bool,
    require_prompts,
    tokenize,
)
from soup_cli.utils.diagnose.report import FailureScore, classify_score

# Generator that emits K completions per prompt; (prompt, k) -> list[str].
MultiGen = Callable[[str, int], Sequence[str]]


def _pairwise_diversity(samples: Sequence[str], *, n: int = 3) -> float | None:
    """1 - average pairwise n-gram-set Jaccard; 1.0 = fully diverse."""
    cleaned = []
    evidence = []

    for sample in samples:
        if not isinstance(sample, str):
            continue

        tokens = tokenize(sample)
        has_word = any(unicodedata.category(ch)[0] in "LN" for ch in sample)

        if not tokens and not has_word:
            tokens = [
                ch
                for ch in sample
                if unicodedata.category(ch)[0] in "SP"
            ]

        cleaned.append(tokens)
        evidence.append(bool(tokens) or has_word)

    if len(cleaned) < 2:
        return None

    pairs = 0
    overlap = 0.0

    for i in range(len(cleaned)):
        for j in range(i + 1, len(cleaned)):
            if not evidence[i] and not evidence[j]:
                continue

            a = ngrams(cleaned[i], n) or [tuple(cleaned[i])]
            b = ngrams(cleaned[j], n) or [tuple(cleaned[j])]

            overlap += jaccard(a, b)
            pairs += 1

    if pairs == 0:
        return None

    return max(0.0, 1.0 - overlap / pairs)


def score_mode_collapse(
    prompts: Sequence[str],
    adapter_multi_gen: MultiGen,
    *,
    k: int = 4,
    ngram_n: int = 3,
) -> FailureScore:
    """Score 1.0 - mean pairwise overlap across K completions per prompt."""
    reject_bool(k, "k")
    if not isinstance(k, int):
        raise TypeError("k must be int")
    if k < 2 or k > 32:
        raise ValueError("k must be in [2, 32]")
    reject_bool(ngram_n, "ngram_n")
    if not isinstance(ngram_n, int):
        raise TypeError("ngram_n must be int")
    if ngram_n < 1 or ngram_n > 8:
        raise ValueError("ngram_n must be in [1, 8]")
    if not callable(adapter_multi_gen):
        raise TypeError("adapter_multi_gen must be callable")
    prompts_list = require_prompts(prompts, max_count=2_000)
    if not prompts_list:
        return FailureScore(
            mode="mode_collapse",
            score=1.0,
            verdict="OK",
            evidence="no prompts; nothing to check",
        )
    diversities = []
    for prompt in prompts_list:
        samples = adapter_multi_gen(prompt, k)
        if not isinstance(samples, Sequence) or isinstance(samples, (str, bytes)):
            raise TypeError("adapter_multi_gen must return a sequence of str")
        div = _pairwise_diversity(samples, n=ngram_n)
        if div is not None:
            diversities.append(div)

    # Neutral score with reason if all pairs were skipped across all prompts
    if not diversities:
        return FailureScore(
            mode="mode_collapse",
            score=1.0,
            verdict="OK",
            evidence="no token evidence in any answer pair; nothing to check",
        )

    score = sum(diversities) / len(diversities)
    score = max(0.0, min(1.0, score))
    verdict = classify_score(score)
    evidence = merge_evidence(
        {
            "prompts": len(prompts_list),
            "k": k,
            "ngram_n": ngram_n,
            "mean_diversity": score,
        }
    )
    return FailureScore(
        mode="mode_collapse",
        score=score,
        verdict=verdict,
        evidence=evidence,
    )