Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
"""Small, reproducible choice-invariance diagnostics (issue #244).
Run from the checkout: python -m research.eval.metamorphic --help.
Pure transformations/metrics can also be used without a model or gold labels.
"""
from __future__ import annotations
import argparse
from copy import deepcopy
from dataclasses import dataclass
import json
import math
import random
from typing import Any, Callable, Sequence
from . import laya_eval as harness
@dataclass
class MetamorphicCase:
"""One canonical choice decision in the existing harness input format."""
state: Any
questions: dict
gold_index: int | None = None
def __post_init__(self):
if len(self.questions) != 1:
raise ValueError("expected exactly one choice question per case")
question = next(iter(self.questions.values()))
criteria = question.get("criteria")
if question.get("type") != "choice" or not isinstance(criteria, dict):
raise ValueError("expected choice criteria as a dictionary")
if len(criteria) < 2:
raise ValueError("at least two options are required")
if any(not isinstance(k, str) or not k.strip() for k in criteria):
raise ValueError("option keys must be nonempty strings")
if self.gold_index is not None and (
type(self.gold_index) is not int or not 0 <= self.gold_index < len(criteria)):
raise ValueError("gold index outside canonical options")
@property
def option_keys(self):
return list(next(iter(self.questions.values()))["criteria"])
def as_pair(self):
return self.state, self.questions
@dataclass
class MetamorphicVariant:
"""Transformed input plus an explicit bidirectional semantic label mapping."""
kind: str
case: MetamorphicCase
canonical_to_transformed: dict[str, str]
@property
def transformed_to_canonical(self):
return {v: k for k, v in self.canonical_to_transformed.items()}
def as_record(self):
mapping = self.canonical_to_transformed
inverse = self.transformed_to_canonical
keys = list(mapping)
if len(inverse) != len(mapping) or set(inverse) != set(self.case.option_keys):
raise ValueError("label mapping must be a bijection over transformed options")
return {"kind": self.kind, "case": self.case.as_pair(),
"canonical_to_transformed": dict(mapping),
"transformed_to_canonical": inverse,
"canonical_indices": [keys.index(inverse[k]) for k in self.case.option_keys]}
def _transform(case, kind, order, labels):
questions = deepcopy(case.questions)
question = next(iter(questions.values()))
criteria = question["criteria"]
keys = case.option_keys
mapping = dict(zip(keys, labels))
question["criteria"] = {mapping[keys[i]]: deepcopy(criteria[keys[i]]) for i in order}
gold = None if case.gold_index is None else order.index(case.gold_index)
return MetamorphicVariant(kind, MetamorphicCase(deepcopy(case.state), questions, gold), mapping)
def permute_options(case: MetamorphicCase, seed: int = harness.SEED):
"""One deterministic nonidentity shuffle; identity falls back to rotation."""
order = list(range(len(case.option_keys)))
random.Random(seed).shuffle(order)
if order == list(range(len(order))):
order = order[1:] + order[:1]
return _transform(case, "option_order", order, case.option_keys)
def opaque_labels(count: int, alphabet: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ") -> list[str]:
"""Deterministic neutral labels: A, B, ... Z, then key_26, key_27, ..."""
if count < 1:
raise ValueError("at least one label is required")
if len(set(alphabet)) < count:
return [f"key_{i}" for i in range(count)]
return list(alphabet[:count])
def rename_labels(case: MetamorphicCase, alphabet: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
"""Replace model-facing labels with neutral opaque ones; semantics and order unchanged.
The option at canonical position i is renamed to opaque_labels[i], so the
presented order is identical and only the label tokens differ (issue #512).
"""
labels = opaque_labels(len(case.option_keys), alphabet)
return _transform(case, "label_rename", list(range(len(case.option_keys))), labels)
def _baseline(case):
return _transform(case, "baseline", list(range(len(case.option_keys))), case.option_keys)
def make_variants(case, rng: random.Random):
"""Adapt the harness tuple format to baseline, option permutation, label rename.
Paraphrases and other metamorphic transforms are intentionally deferred.
"""
canonical = MetamorphicCase(*case)
return [v.as_record() for v in (
_baseline(canonical), permute_options(canonical, rng.getrandbits(64)),
rename_labels(canonical))]
def canonicalize(probabilities, canonical_indices):
"""Validate a probability vector and restore canonical semantic ordering."""
values = [float(p) for p in probabilities]
n = len(values)
if not n or any(type(i) is not int for i in canonical_indices) or sorted(canonical_indices) != list(range(n)):
raise ValueError("probabilities and canonical mapping must be a bijection")
if any(not math.isfinite(p) or p < 0 or p > 1 for p in values):
raise ValueError("probabilities must be finite values in [0, 1]")
if not math.isclose(sum(values), 1.0, abs_tol=1e-6, rel_tol=0):
raise ValueError("probabilities must sum to one")
restored = [0.0] * n
for p, index in zip(values, canonical_indices):
restored[index] = p
return restored
def distribution_metrics(baseline, variant):
"""Drift in canonical space; JS uses natural logs (range 0..ln(2)).
Ties choose the first canonical option, so reordering tied slots does not
itself count as disagreement. Confidence is the maximum probability;
confidence drift is signed (variant minus baseline).
"""
p = canonicalize(baseline, range(len(baseline)))
q = canonicalize(variant, range(len(baseline)))
middle = [(a + b) / 2 for a, b in zip(p, q)]
js = sum(0.5 * x * math.log(x / m)
for distribution in (p, q)
for x, m in zip(distribution, middle) if x > 0)
drift = [abs(a - b) for a, b in zip(p, q)]
return {
"semantic_agreement": max(range(len(p)), key=p.__getitem__) ==
max(range(len(q)), key=q.__getitem__),
"mean_probability_drift": sum(drift) / len(drift),
"max_probability_drift": max(drift),
"js_divergence": max(0.0, js),
"confidence_drift": max(q) - max(p),
}
def summarise_pairs(pairs):
"""Pair-weighted summary; absent observations are not perfect agreement."""
if not pairs:
return {"n": 0}
return {
"n": len(pairs),
"semantic_agreement_rate": sum(p["semantic_agreement"] for p in pairs) / len(pairs),
"mean_probability_drift": sum(p["mean_probability_drift"] for p in pairs) / len(pairs),
"max_probability_drift": max(p["max_probability_drift"] for p in pairs),
"mean_js_divergence": sum(p["js_divergence"] for p in pairs) / len(pairs),
"mean_confidence_drift": sum(p["confidence_drift"] for p in pairs) / len(pairs),
"mean_absolute_confidence_drift": sum(abs(p["confidence_drift"]) for p in pairs) / len(pairs),
"worst_confidence_increase_on_disagreement": max(
[0.0] + [p["confidence_drift"] for p in pairs if not p["semantic_agreement"]]),
}
def evaluate(cases, score: Callable, gold_indices=None, seed=harness.SEED,
batch_size=16, budget: BudgetProbe | None = None):
"""Score cases via ``score(batch) -> probability vectors in presented order``.
Optional gold indices refer to original option order; entries may be None.
Batches bound inference memory. RNG state and variants do not depend on batch
size. Returned records retain inputs, mappings and full precision vectors.
An optional ``budget`` probe checks whether both sides of each comparison still
carry the same rendered option information (#543/#517/#569).
"""
cases = list(cases)
rng = random.Random(seed)
generated = [make_variants(case, rng) for case in cases]
return _evaluate_generated(cases, generated, score, gold_indices, batch_size, budget)
def _evaluate_generated(cases, generated_by_case, score, gold_indices, batch_size,
budget=None):
if batch_size < 1:
raise ValueError("batch_size must be positive")
cases = list(cases)
golds = [None] * len(cases) if gold_indices is None else list(gold_indices)
if len(golds) != len(cases):
raise ValueError("one gold index is required per case")
variants, records = [], []
for index, (case, gold) in enumerate(zip(cases, golds)):
generated = generated_by_case[index]
keys = list(next(iter(case[1].values()))["criteria"])
if gold is not None and (type(gold) is not int or not 0 <= gold < len(keys)):
raise ValueError("gold index outside canonical options")
records.append({"index": index, "state": deepcopy(case[0]),
"questions": deepcopy(case[1]), "options": keys,
"gold_index": gold, "variants": []})
variants.extend((index, v) for v in generated)
for start in range(0, len(variants), batch_size):
batch = variants[start:start + batch_size]
vectors = list(score([v["case"] for _, v in batch]))
if len(vectors) != len(batch):
raise ValueError("scorer returned the wrong number of probability vectors")
for (index, v), raw in zip(batch, vectors):
probs = canonicalize(raw, v["canonical_indices"])
pred = max(range(len(probs)), key=probs.__getitem__)
record = {
"kind": v["kind"],
"presented_options": list(next(iter(v["case"][1].values()))["criteria"]),
"canonical_indices": v["canonical_indices"],
"canonical_to_transformed": v["canonical_to_transformed"],
"transformed_to_canonical": v["transformed_to_canonical"],
"probabilities": probs, "pred_index": pred,
"pred_label": records[index]["options"][pred],
"confidence": max(probs),
"correct": None if golds[index] is None else pred == golds[index],
}
if budget is not None:
record["budget"] = budget.measure(*v["case"])
records[index]["variants"].append(record)
return {"report": _report(records), "cases": records}
def auroc(scores, labels):
"""Chance that a random correct case scores above a random wrong one.
Ties count half. Returns None unless both correct and wrong cases exist.
"""
pairs = sorted(zip(scores, labels), key=lambda pair: pair[0])
ranks, start = [0.0] * len(pairs), 0
while start < len(pairs):
end = start
while end + 1 < len(pairs) and pairs[end + 1][0] == pairs[start][0]:
end += 1
for i in range(start, end + 1):
ranks[i] = (start + end) / 2 + 1
start = end + 1
positives = sum(1 for _, label in pairs if label)
negatives = len(pairs) - positives
if not positives or not negatives:
return None
rank_sum = sum(rank for rank, (_, label) in zip(ranks, pairs) if label)
return (rank_sum - positives * (positives + 1) / 2) / (positives * negatives)
def selective_prediction(records, coverages=(0.5, 0.7, 0.8, 0.9)):
"""Whether disagreement under the transforms predicts which baseline answers are wrong.
Per labelled case, with w the baseline winner:
confidence baseline probability of w
agreement_<kind> 1 if the <kind> variant(s) also pick w, else the share that do
support_<kind> mean probability the <kind> variant(s) give w
support_all mean probability of w over the baseline and every variant
Reports AUROC per signal and, for the continuous signals, the accuracy of the
most-trusted fraction of cases at each coverage (ties broken by case order).
"""
rows = []
for record in records:
baseline, variants = record["variants"][0], record["variants"][1:]
if baseline.get("correct") is None:
continue
winner = baseline["pred_index"]
row = {"correct": bool(baseline["correct"]), "confidence": baseline["probabilities"][winner]}
for kind in dict.fromkeys(v["kind"] for v in variants):
same = [v for v in variants if v["kind"] == kind]
row["agreement_" + kind] = sum(v["pred_index"] == winner for v in same) / len(same)
row["support_" + kind] = sum(v["probabilities"][winner] for v in same) / len(same)
row["support_all"] = sum(v["probabilities"][winner] for v in record["variants"]) / len(record["variants"])
rows.append(row)
wrong = sum(not r["correct"] for r in rows)
if not rows:
return {"n_labelled": 0}
signals = [key for key in rows[0] if key != "correct" and all(key in r for r in rows)]
labels = [r["correct"] for r in rows]
result = {"n_labelled": len(rows), "n_wrong": wrong,
"auroc": {key: auroc([r[key] for r in rows], labels) for key in signals},
"accuracy_at_coverage": {}}
for key in signals:
if key.startswith("agreement_"):
continue # near-binary: the coverage cut would mostly depend on tie order
ranked = sorted(range(len(rows)), key=lambda i: -rows[i][key])
result["accuracy_at_coverage"][key] = {
str(c): sum(rows[i]["correct"] for i in ranked[:max(1, round(len(rows) * c))])
/ max(1, round(len(rows) * c)) for c in coverages}
return result
def _report(records):
groups = {"option_order": [], "label_rename": []}
budget_groups = {"option_order": [], "label_rename": []}
for record in records:
baseline = record["variants"][0]
for variant in record["variants"][1:]:
variant["comparison"] = distribution_metrics(
baseline["probabilities"], variant["probabilities"])
groups[variant["kind"]].append(variant["comparison"])
if "budget" in baseline and "budget" in variant:
variant["budget_comparison"] = compare_budgets(
baseline["budget"], variant["budget"], variant["canonical_indices"],
kind=variant["kind"])
budget_groups[variant["kind"]].append(variant["budget_comparison"])
report = {kind: summarise_pairs(pairs) for kind, pairs in groups.items()}
report["overall"] = summarise_pairs([p for pairs in groups.values() for p in pairs])
if any(budget_groups.values()):
report["budget"] = {kind: summarise_budgets(pairs)
for kind, pairs in budget_groups.items()}
report["budget"]["overall"] = summarise_budgets(
[p for pairs in budget_groups.values() for p in pairs])
report["quality"] = {}
for kind in ("baseline", "option_order", "label_rename"):
labelled = [v for r in records for v in r["variants"]
if v["kind"] == kind and v["correct"] is not None]
quality = {"n_labelled": len(labelled)}
if labelled:
quality.update(accuracy=sum(v["correct"] for v in labelled) / len(labelled),
ece=harness.ece([v["confidence"] for v in labelled],
[v["correct"] for v in labelled]))
report["quality"][kind] = quality
report["selective_prediction"] = selective_prediction(records)
return report
def model_scorer(agent, unclamped=False):
"""Reuse the harness's raw logits and per-option-count temperatures."""
from laya.common import QTYPES
def score(cases):
return [harness.softmax_t(z, harness.temperature_for(
agent, QTYPES["choice"], len(z), unclamped))
for z in harness.score_cases(agent, cases)]
return score
_ATTRIBUTION_LIMIT = 4096
"""Longest option text (characters) whose surviving span is attributed at all."""
def _retained_prefix(encode, text, kept, full):
"""Character prefix of `text` the surviving token span `kept` stands for, or None.
The only sound claim is an exact witness: a prefix whose encoding equals `kept`
token for token. Prefix token counts are not guaranteed to be non-decreasing --
a BPE merge can encode a longer prefix to fewer tokens, and lossy normalization
can drop characters -- so no bisection point identifies the witness, witnesses
need not be adjacent, and a single exact witness is required: with several, which
characters the model kept is undecidable. Every prefix is therefore checked, and
anything above `_ATTRIBUTION_LIMIT` characters is left unattributed rather than
searched.
None means the span could not be attributed to a character boundary, which the
caller must not read as "nothing was lost".
"""
if full[:len(kept)] != kept:
return None
if full == kept:
return text
if len(text) > _ATTRIBUTION_LIMIT:
return None
witnesses = [size for size in range(len(text)) if encode(text[:size]) == kept]
return text[:witnesses[0]] if len(witnesses) == 1 else None
def _retained_description(retained, label, rendered):
"""Description characters a truncated option kept, or None when unverifiable."""
if retained is None or not rendered.startswith(retained):
return None
marker = label + ": "
start = len(marker) if rendered.startswith(marker) else len(rendered)
return rendered[start:len(retained)]
def _description_present(value):
"""Whether a criterion value carries description text at all.
None and blank strings mean "no description" -- whitespace renders as blanks, so
it is missing too. Anything structured (dict, list, number, boolean) renders as
JSON text and counts as present; `0` and `False` are legitimate criterion values
(see `render_options`), not absences.
"""
if value is None:
return False
from laya.common import render_criterion
return render_criterion(value).strip() != ""
def _to_canonical(values, canonical_indices):
"""Restore presented-order values to canonical order.
`canonical_indices[p]` is the canonical slot of the option presented at position
`p` (the same direction `canonicalize` uses for probabilities), so the inverse
assignment is a scatter, not a gather: gathering is only correct when the
permutation is its own inverse.
"""
restored = [None] * len(values)
for presented, canonical in enumerate(canonical_indices):
restored[canonical] = values[presented]
return restored
def _canonical_partition(classes, canonical_indices):
"""Span classes relabelled by first canonical occurrence, so permutations compare equal."""
relabel = {}
return [relabel.setdefault(class_, len(relabel))
for class_ in _to_canonical(classes, canonical_indices)]
class BudgetProbe:
"""Renders options exactly as the model sees them and reports what survived.
A metamorphic comparison is only information-equivalent if both sides still present
the same option content after rendering and budgeting; this probe measures that per
pair. It follows the option-collapse question of #543 (closed; addressed at runtime
in v0.3.21 via #569/#542) on the evaluation side, where #569's per-inference report
cannot answer whether baseline and transformed variants agree.
`build_sequence` caps every option and, when the head budget runs out, re-caps them
all to the same length, so a shorter label leaves more room for its description. The
probe reads the option spans back from one `build_sequence` call with an empty state,
which closes the last option's span -- in a scored sequence it runs on into the
serialized state and always looks distinguishable (#538) -- and checks them against
the stats that same call returns.
"""
def __init__(self, tok, max_len=512, head_max_len=192):
self.tok = tok
self.max_len = max_len
self.head_max_len = head_max_len
@classmethod
def from_agent(cls, agent):
cfg = getattr(agent, "cfg", None) or {}
return cls(agent.tok, cfg.get("max_len", 512), cfg.get("head_max_len", 192))
def measure(self, state, questions):
"""One case's option spans, span classes and retained descriptions.
Returns {"error": ...} instead of raising: a tokenizer the probe cannot drive,
or a question the budget cuts apart, has to leave the comparison unverifiable
rather than decide it.
"""
try:
return self._measure(state, questions)
except Exception as exc:
return {"error": "%s: %s" % (type(exc).__name__, exc)}
def _measure(self, state, questions):
from laya.common import build_sequence, encode_text, render_options
qdef = next(iter(questions.values()))
q = harness.internal_question(qdef)
ids, markers, usage = build_sequence(
self.tok, state, q, self.max_len, self.head_max_len, state_ids=[], return_stats=True)
rendered = render_options(q)
if len(markers) != len(rendered):
# score_cases raises on the same condition: options the head budget accepted
# were dropped again by max_len, so these are not the spans that get scored.
raise ValueError("markers %d != options %d" % (len(markers), len(rendered)))
sep = self.tok.sep_token_id
# With an empty state the last two ids are the option-block and state separators;
# anything else at max_len means max_len truncated the option block itself.
tail_truncated = len(ids) == self.max_len and ids[-2:] != [sep, sep]
spans = [ids[markers[i] + 1:markers[i + 1]] for i in range(len(markers) - 1)]
spans.append(ids[markers[-1] + 1:] if tail_truncated else ids[markers[-1] + 1:-2])
distinct = len({tuple(span) for span in spans})
if not tail_truncated and distinct != usage["options_distinct"]:
raise ValueError("recovered spans disagree with build_sequence stats")
def encode(part):
return encode_text(self.tok, " " + part, add_special_tokens=False)["input_ids"]
labels = list(qdef["criteria"])
description_present = [_description_present(value) for value in qdef["criteria"].values()]
classes, class_of, retained_descriptions = [], {}, []
retained_characters = []
truncated = False
for position, span in enumerate(spans):
text = rendered[position].replace(self.tok.mask_token, " ")
full = encode(text)
truncated = truncated or len(span) < len(full)
retained = _retained_prefix(encode, text, list(span), full)
retained_characters.append(retained)
retained_descriptions.append(_retained_description(retained, labels[position], text))
key = tuple(span)
if key not in class_of:
class_of[key] = len(class_of)
classes.append(class_of[key])
return {
"options": len(spans),
"distinct_spans": distinct,
"tokens_per_option": usage["tokens_per_option"],
"instruction_tokens": markers[0] - 2,
"span_classes": classes,
"retained_text": retained_characters,
"retained_descriptions": retained_descriptions,
"description_present": description_present,
"truncated": truncated,
"tail_truncated": tail_truncated,
}
def compare_budgets(baseline, variant, canonical_indices, *, kind):
"""Whether both sides of a comparison still carry the same option information.
`baseline` and `variant` are `BudgetProbe.measure` results; the baseline must be in
canonical order, and `canonical_indices` maps the variant's presented slots back to
canonical ones (the direction `canonicalize` uses). `kind` is the comparison's
transformation: a `label_rename` over options whose descriptions are missing removes
the only semantic content there was, so its verdict is unknown regardless of what the
rendering did; an `option_order` permutation moves every key/value pair together and
stays valid. `budget_confounded` is True (the rendering no longer carries the same
content, so drift cannot be read as lexical sensitivity alone), False, or None when
the rendering could not be measured or attributed. `tokens_per_option` is reported
but never confounds by itself: when the re-cap bites it changes the spans reported
here, and when it does not the model sees the same sequence either way.
"""
comparison = {key: {"baseline": baseline.get(key), "variant": variant.get(key)}
for key in ("options", "distinct_spans", "tokens_per_option",
"instruction_tokens")}
errors = {side: measurement["error"] for side, measurement in
(("baseline", baseline), ("variant", variant)) if "error" in measurement}
if errors:
return {**comparison, "retained_descriptions": None, "budget_confounded": None,
"reasons": [{"code": "measurement_error", "errors": errors}]}
baseline_retained = baseline["retained_descriptions"]
variant_retained = _to_canonical(variant["retained_descriptions"], canonical_indices)
comparison["retained_descriptions"] = {"baseline": baseline_retained,
"variant": variant_retained}
reasons = []
if baseline["options"] != variant["options"]:
reasons.append({"code": "option_count"})
if baseline["distinct_spans"] != variant["distinct_spans"]:
reasons.append({"code": "distinct_spans"})
if baseline["instruction_tokens"] != variant["instruction_tokens"]:
reasons.append({"code": "instruction_tokens"})
if baseline["tail_truncated"] or variant["tail_truncated"]:
reasons.append({"code": "tail_truncation"})
if baseline["options"] == variant["options"] and _canonical_partition(
baseline["span_classes"], range(len(baseline["span_classes"]))) != (
_canonical_partition(variant["span_classes"], canonical_indices)):
reasons.append({"code": "collision_partition"})
pairs = list(zip(baseline_retained, variant_retained))
unverified = [slot for slot, pair in enumerate(pairs) if None in pair]
differing = [slot for slot, pair in enumerate(pairs)
if None not in pair and pair[0] != pair[1]]
if differing:
reasons.append({"code": "retained_description", "slots": differing})
if kind == "label_rename":
# Descriptions are moved to the same canonical slots by the rename, so the two
# sides' presence vectors are compared as-is once the variant is restored. The
# model observation is kept; only the experiment's validity is unknown.
presence = _to_canonical(variant["description_present"], canonical_indices)
comparison["description_present"] = {"baseline": baseline["description_present"],
"variant": presence}
missing = [slot for slot, (base, var) in enumerate(
zip(baseline["description_present"], presence)) if not base or not var]
if missing:
return {**comparison, "budget_confounded": None,
"reasons": [{"code": "missing_semantic_description", "slots": missing}]}
if reasons:
return {**comparison, "budget_confounded": True, "reasons": reasons}
if unverified:
return {**comparison, "budget_confounded": None,
"reasons": [{"code": "unverified_retained_text", "slots": unverified}]}
return {**comparison, "budget_confounded": False, "reasons": [{"code": "clean"}]}
def summarise_budgets(pairs):
"""Budget verdict counts per group; `confounded_rate` is over verifiable pairs only."""
if not pairs:
return {"n": 0}
verifiable = [pair for pair in pairs if pair["budget_confounded"] is not None]
return {
"n": len(pairs),
"clean": sum(pair["budget_confounded"] is False for pair in pairs),
"confounded": sum(pair["budget_confounded"] is True for pair in pairs),
"unknown": sum(pair["budget_confounded"] is None for pair in pairs),
"confounded_rate": (sum(pair["budget_confounded"] is True for pair in verifiable)
/ len(verifiable) if verifiable else None),
}
@dataclass
class MetamorphicResults:
baseline: dict
variants: list[dict]
def evaluate_variants(agent, case: MetamorphicCase, variants, *, score=None,
unclamped=False, batch_size=16, budget=None):
"""Evaluate explicit transforms; inject ``score`` for offline stub models.
With a real agent, call ``agent.model.eval()`` first, as with the harness.
All output vectors are restored to the original case's canonical ordering.
A real agent is probed for the option budget by default; a stub ``score``
is left unprobed unless ``budget`` is passed explicitly.
"""
probe = budget if budget is not None else (BudgetProbe.from_agent(agent) if score is None else None)
generated = [_baseline(case).as_record()]
for variant in variants:
if variant.kind not in ("option_order", "label_rename"):
raise ValueError("unsupported transformation kind")
if list(variant.canonical_to_transformed) != case.option_keys:
raise ValueError("variant mapping does not match the canonical case")
generated.append(variant.as_record())
result = _evaluate_generated([case.as_pair()], [generated],
score if score is not None else model_scorer(agent, unclamped),
[case.gold_index], batch_size, probe)
predictions = result["cases"][0]["variants"]
return MetamorphicResults(predictions[0], predictions[1:])
def compare_predictions(*, baseline, variants):
"""Report semantic agreement separately from continuous stability metrics.
No drift threshold or automatic bug classification is applied.
"""
return _report([{"variants": [deepcopy(baseline), *deepcopy(variants)]}])
def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--model", default="convaiinnovations/laya")
parser.add_argument("--subfolder", default=None)
parser.add_argument("--device", default=None)
parser.add_argument("--langs", default="en", help="comma-separated MASSIVE configs, or all")
parser.add_argument("--per-lang", type=int, default=harness.PER_LANG)
parser.add_argument("--n-opts", type=int, default=harness.N_OPTS)
parser.add_argument("--seed", type=int, default=harness.SEED)
parser.add_argument("--batch-size", type=int, default=16)
parser.add_argument("--unclamped", action="store_true")
parser.add_argument("--out", required=True, help="JSON report path")
args = parser.parse_args(argv)
if args.per_lang < 1 or args.n_opts < 2 or args.batch_size < 1:
parser.error("per-lang/batch-size must be positive and n-opts must be at least 2")
langs = (harness.available_languages() if args.langs.strip().lower() == "all"
else list(dict.fromkeys(x.strip() for x in args.langs.split(",") if x.strip())))
if not langs:
parser.error("no languages selected")
import laya
agent = laya.load(args.model, device=args.device, subfolder=args.subfolder)
agent.model.eval()
probe = BudgetProbe.from_agent(agent)
payload: dict[str, Any] = {
"config": {**vars(args), "dataset": harness.DATASET, "split": "test",
"device": str(agent.device), "laya_version": laya.__version__,
"max_len": agent.cfg.get("max_len"),
"head_max_len": agent.cfg.get("head_max_len"),
"temperature": list(agent.temperature_raw if args.unclamped else agent.temperature),
"temperature_by_options": dict(agent.temperature_by_options_raw if args.unclamped
else agent.temperature_by_options),
"permutations_per_case": 1, "label_renames_per_case": 1, "js_log_base": "e"},
"report": {}, "cases": [],
}
failed = False
for lang in langs:
try:
rows = harness.load_language(lang)
cases, gold, _ = harness.build_suite(
rows, sorted({r["label_text"] for r in rows}), args.per_lang, args.n_opts, args.seed)
if not cases:
raise ValueError("dataset returned no cases")
result = evaluate(cases, model_scorer(agent, args.unclamped), gold,
args.seed, args.batch_size, probe)
payload["report"][lang] = result["report"]
payload["cases"].extend({"lang": lang, **r} for r in result["cases"])
print(lang, json.dumps(result["report"]), flush=True)
except Exception as exc:
failed = True
payload["report"][lang] = {"error": str(exc)}
print("%s FAILED: %s" % (lang, exc), flush=True)
with open(args.out, "w", encoding="utf-8") as stream:
json.dump(payload, stream, ensure_ascii=False, indent=2, allow_nan=False)
stream.write("\n")
return 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())