Skip to content

Code Video Model Official Repo

JavaScript
9.9 KB
Raw Blame
"""Evidence-first judgment validation and non-compensating score aggregation."""
import json
import math
from pathlib import Path
import unicodedata

PROMPTS = Path(__file__).resolve().parents[1] / "prompts/evaluation"
SYSTEM = (PROMPTS / "system.txt").read_text(encoding="utf-8")
RULES = {axis: (PROMPTS / f"{axis.lower()}.txt").read_text(encoding="utf-8")
         for axis in ("Structure", "Style", "Quality")}
SEVERITY_CAPS = {'none': 100, 'minor': 89, 'moderate': 69, 'major': 39, 'critical': 19}


def aggregate(subscores, checks, axis, weights):
    mean = sum(subscores[key] * weight for key, weight in weights[axis].items())
    score, caps = mean, []
    if axis == "Style":
        score = min(score, subscores["photorealism"])
        if score < mean:
            caps.append("Style cannot exceed photorealism")
    elif axis in ("Structure", "Quality"):
        severe = [SEVERITY_CAPS[c["severity"]] for c in checks.values()
                  if c["severity"] in ("major", "critical")]
        if severe and score > min(severe):
            score = min(severe)
            caps.append("Central major/critical failure ceiling")
    return round(score, 2), round(mean, 2), caps


def check_timestamp(value, duration):
    if type(value) not in (int, float) or not math.isfinite(value) or value < 0:
        raise ValueError("Evidence timestamp is not a valid non-negative number")
    if value <= duration:
        return False
    if value - duration <= .050001 and abs(value * 10 - round(value * 10)) < 1e-8:
        return True
    raise ValueError("Evidence timestamp outside video beyond endpoint rounding")


def validate(raw, axis, weights, metadata, prompt=None):
    text = raw.rsplit("</think>", 1)[-1].strip()
    if text.startswith("```json") and text.endswith("```"):
        text = text[7:-3].strip()
    data = json.loads(text)
    required = {"subscores", "checks", "evidence", "confidence", "limitations"}
    if not required <= set(data):
        raise ValueError("Evidence-first schema fields are missing or unexpected")
    schema_flags = []
    if set(data) - required:
        schema_flags.append("Additional top-level metadata retained but excluded from score calculation")
    keys = set(weights[axis])
    if set(data["subscores"]) != keys or set(data["checks"]) != keys:
        raise ValueError("Criterion keys do not match the frozen rubric")
    if data["confidence"] not in ("low", "medium", "high"):
        raise ValueError("Invalid confidence")
    if not isinstance(data["limitations"], str) or not data["limitations"].strip():
        raise ValueError("Explicit limitations are required")
    evidence = data["evidence"]
    if not isinstance(evidence, list) or not evidence:
        raise ValueError("At least opening/middle/ending evidence is required")
    repaired = []
    evidence_repairs = []
    index = 0
    while index < len(evidence):
        entry = evidence[index]
        if (entry == "evidence_type" and index + 1 < len(evidence)
                and evidence[index + 1] in ("frame", "interval")
                and repaired and isinstance(repaired[-1], dict)
                and "evidence_type" not in repaired[-1]):
            repaired[-1]["evidence_type"] = evidence[index + 1]
            evidence_repairs.append({"kind": "detached_evidence_type_pair", "observation": len(repaired) - 1,
                                     "value": evidence[index + 1]})
            index += 2
            continue
        if not isinstance(entry, dict):
            raise ValueError("Unrecognized non-object evidence; not guessing its meaning")
        repaired.append(entry)
        index += 1
    evidence = data["evidence"] = repaired
    if evidence_repairs:
        data["evidence_repairs"] = evidence_repairs
    if {e.get("phase") for e in evidence} != {"opening", "middle", "ending"} or len(evidence) < 3:
        schema_flags.append("Evidence does not cover all three required phases; temporal review required")
    combined_style_evidence = False
    timestamp_notes = []
    quote_flags = []
    for number, event in enumerate(evidence):
        if (axis == "Style" and "observation" not in event
                and isinstance(event.get("reference_observation"), str)
                and "candidate" in event["reference_observation"].lower()):
            # The response supplied a combined comparison rather than separate
            # fields. Preserve the exact text and make the normalization explicit.
            event["observation"] = event["reference_observation"]
            combined_style_evidence = True
        expected = {"phase", "seconds", "observation"}
        if axis == "Structure":
            expected |= {"source_seconds", "source_observation", "prompt_clause"}
        elif axis == "Style":
            expected.add("reference_observation")
        if not expected <= set(event):
            raise ValueError("Evidence fields do not match this dimension")
        if set(event) - expected - {"evidence_type", "index", "id"}:
            schema_flags.append(f"evidence {number}: additional metadata retained without using it for scoring")
        if event.get("evidence_type", "frame") not in ("frame", "interval"):
            raise ValueError("Unknown evidence type")
        if "index" in event and (type(event["index"]) is not int or event["index"] < 0):
            raise ValueError("Invalid optional visual-input index")
        if "id" in event and (type(event["id"]) is not int or event["id"] < 0):
            raise ValueError("Invalid optional evidence identifier")
        if check_timestamp(event["seconds"], metadata["candidate"]["duration"]):
            timestamp_notes.append({"evidence": number, "role": "candidate", "reported_seconds": event["seconds"],
                                    "interpretation": "one-decimal clip-end rounding; original timestamp retained"})
        for field in expected - {"phase", "seconds", "source_seconds", "prompt_clause"}:
            if not isinstance(event[field], str) or not event[field].strip():
                raise ValueError("Specific non-empty visual observations required")
        if axis == "Structure":
            if check_timestamp(event["source_seconds"], metadata["threejs"]["duration"]):
                timestamp_notes.append({"evidence": number, "role": "source", "reported_seconds": event["source_seconds"],
                                        "interpretation": "one-decimal clip-end rounding; original timestamp retained"})
            quote = event["prompt_clause"]
            if quote is not None:
                if not isinstance(quote, str) or not quote.strip():
                    raise ValueError("Prompt clause must be a nonempty string or null")
                if prompt is not None and quote not in prompt:
                    normalized_quote = " ".join(unicodedata.normalize("NFKC", quote).casefold().split())
                    normalized_prompt = " ".join(unicodedata.normalize("NFKC", prompt).casefold().split())
                    kind = "format/case variation" if normalized_quote in normalized_prompt else "unverified or paraphrased citation"
                    quote_flags.append(f"evidence {number}: prompt quote is not verbatim ({kind}); original wording retained")
    consistency_flags = []
    consistency_flags.extend(schema_flags)
    consistency_flags.extend(quote_flags)
    if evidence_repairs:
        consistency_flags.append("Detached optional evidence metadata was repaired; raw answer retained")
    if combined_style_evidence:
        data["evidence_normalization"] = "combined_reference_and_candidate_text_preserved_in_both_fields"
        consistency_flags.append("Style observations were combined rather than independently described")
    all_indices = [i for check in data["checks"].values() for i in check.get("evidence_indices", [])]
    one_based = (bool(all_indices) and all(type(i) is int and 1 <= i <= len(evidence) for i in all_indices)
                 and len(evidence) in all_indices)
    if one_based:
        # A uniformly one-based citation convention can be converted without
        # altering observations, subscores or the saved raw API answer.
        data["original_evidence_indices"] = {k: list(c["evidence_indices"]) for k, c in data["checks"].items()}
        for check in data["checks"].values():
            check["evidence_indices"] = [i - 1 for i in check["evidence_indices"]]
        data["citation_normalization"] = "uniform_one_based_to_zero_based"
    for key, value in data["subscores"].items():
        check = data["checks"][key]
        if set(check) != {"severity", "reason", "evidence_indices"}:
            raise ValueError("Every criterion needs severity/reason/evidence links")
        if check["severity"] not in SEVERITY_CAPS:
            raise ValueError("Unknown severity")
        if type(value) not in (int, float) or not math.isfinite(value) or not 0 <= value <= 100:
            raise ValueError("Subscore must be a finite number in [0,100]")
        if value > SEVERITY_CAPS[check["severity"]]:
            consistency_flags.append(f"{key}: score exceeds its declared {check['severity']} severity band")
        if not isinstance(check["reason"], str) or not check["reason"].strip():
            raise ValueError("Criterion-specific reason is required")
        if not isinstance(check["evidence_indices"], list) or not check["evidence_indices"]:
            raise ValueError("Uncited criterion")
        if any(type(i) is not int or not 0 <= i < len(evidence) for i in check["evidence_indices"]):
            consistency_flags.append(f"{key}: unresolved evidence indices; retained without guessing")
    if axis == "Structure" and not any(e["prompt_clause"] for e in evidence):
        consistency_flags.append("Prompt alignment has no explicit quoted clause; requires review")
    data["score"], data["uncapped_score"], data["score_caps"] = aggregate(data["subscores"], data["checks"], axis, weights)
    data["consistency_flags"] = consistency_flags
    data["timestamp_notes"] = timestamp_notes
    data["needs_review"] = data["confidence"] == "low" or bool(consistency_flags)
    return data