Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

6.3 KB
Raw Blame
"""v0.61.0 Part B — `soup eval unlearning` subcommand.

Computes Forget Quality + Model Utility + PrivLeak metrics from
operator-supplied evidence JSON and emits an :class:`UnlearnReport`.
Scores come from the evidence file given with ``--evidence``; there is no
model-driven evaluation. With no evidence it writes a neutral OK report,
so the schema + output shape are usable.
"""

from __future__ import annotations

import json
from typing import Optional

import typer
from rich.console import Console
from rich.markup import escape
from rich.table import Table

from soup_cli.utils.terminal import for_terminal


def register(app: typer.Typer, console: Console) -> None:
    """Attach v0.61.0 subcommands to ``app``."""

    @app.command(name="unlearning")
    def unlearning_cmd(
        run_id: str = typer.Argument(
            ..., help="Run identifier (e.g. registry id of the unlearn run).",
        ),
        benchmark: str = typer.Option(
            "tofu", "--benchmark", "-b",
            help="Benchmark: tofu / muse / wmdp.",
        ),
        evidence: Optional[str] = typer.Option(
            None, "--evidence", "-e",
            help=(
                "Path to a JSON file with pre-computed evidence. Schema: "
                "{forget_quality: {pre_loss, post_loss}, "
                "model_utility: {pre_acc, post_acc}, "
                "priv_leak: {mia_auc}}. Missing keys fall through to a "
                "neutral OK score."
            ),
        ),
        output: Optional[str] = typer.Option(
            None, "--output", "-o",
            help="Where to write the rendered UnlearnReport JSON.",
        ),
        attach_to_registry: Optional[str] = typer.Option(
            None, "--attach-to-registry",
            help=(
                "Registry entry id to attach the --output report to as an "
                "eval_results artifact (needs --output). Exits 1 when the "
                "attach fails."
            ),
        ),
    ) -> None:
        """Score an unlearn run on TOFU / MUSE / WMDP."""
        from soup_cli.utils.unlearning_eval import (
            load_evidence_file,
            run_unlearn_eval,
            validate_benchmark_name,
            write_unlearn_report,
        )

        try:
            bench = validate_benchmark_name(benchmark)
        except (TypeError, ValueError) as exc:
            console.print(f"[red]Invalid benchmark:[/] {for_terminal(exc)}")
            raise typer.Exit(2) from exc

        # The attach links the --output file, so without one nothing can be
        # attached: refuse before the eval runs.
        if attach_to_registry is not None and output is None:
            console.print(
                "[red]--attach-to-registry requires --output[/] "
                "(nothing written to attach)."
            )
            raise typer.Exit(2)

        evidence_data = None
        if evidence is not None:
            try:
                evidence_data = load_evidence_file(evidence)
            except json.JSONDecodeError as exc:
                # json.JSONDecodeError is a ValueError subclass — catch it
                # FIRST so we can label the message specifically (review
                # MEDIUM M10 — was unreachable after the broader except).
                console.print(
                    f"[red]Evidence file is not valid JSON:[/] {for_terminal(exc)}"
                )
                raise typer.Exit(2) from exc
            except (FileNotFoundError, ValueError, TypeError, OSError) as exc:
                console.print(f"[red]Cannot read evidence:[/] {for_terminal(exc)}")
                raise typer.Exit(2) from exc

        try:
            report = run_unlearn_eval(
                run_id=run_id,
                benchmark=bench,
                evidence=evidence_data,
            )
        except (TypeError, ValueError) as exc:
            console.print(f"[red]Eval failed:[/] {for_terminal(exc)}")
            raise typer.Exit(2) from exc

        # Render the table.
        table = Table(title=f"Unlearn eval — {escape(bench)} — {for_terminal(run_id)}")
        table.add_column("metric", style="bold")
        table.add_column("score", justify="right")
        table.add_column("verdict")
        table.add_column("evidence")
        for m in report.metrics:
            colour = {
                "OK": "green",
                "MINOR": "yellow",
                "MAJOR": "red",
            }.get(m.verdict, "white")
            table.add_row(
                escape(m.name),
                f"{m.score:.3f}",
                f"[{colour}]{escape(m.verdict)}[/]",
                for_terminal(m.evidence),
            )
        console.print(table)
        overall_colour = {
            "OK": "green",
            "MINOR": "yellow",
            "MAJOR": "red",
        }.get(report.overall, "white")
        console.print(
            f"Overall: [{overall_colour}]{escape(report.overall)}[/]"
        )

        # Write the report.
        if output is not None:
            try:
                write_unlearn_report(report, output)
            except (TypeError, ValueError, OSError) as exc:
                console.print(f"[red]Cannot write report:[/] {for_terminal(exc)}")
                raise typer.Exit(2) from exc
            console.print(f"Wrote report -> {for_terminal(output)}")

        # Optional registry attach. Once requested it is part of the command's
        # success: a failure exits 1, with the report already on disk.
        if attach_to_registry is not None:
            assert output is not None  # narrowed by the early arg-check
            try:
                from soup_cli.registry.attach import attach_artifact

                attach_artifact(
                    entry_id=attach_to_registry,
                    path=output,
                    kind="eval_results",
                )
            except Exception as exc:  # noqa: BLE001
                console.print(f"[red]Registry attach failed:[/] {for_terminal(exc)}")
                raise typer.Exit(1) from exc
            console.print(
                f"Attached eval_results -> registry {for_terminal(attach_to_registry)}"
            )

        # Exit code: 0 on OK / MINOR; 2 on MAJOR (matches v0.56.0 diagnose
        # gate convention so CI scripts can chain `soup train` → `soup
        # eval unlearning` → exit-on-MAJOR).
        if report.overall == "MAJOR":
            raise typer.Exit(2)