Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

1.1 KB
Raw Blame
"""Assert decoded parity across benchmark logs that include answers.

python benchmarks/compare_compile_defaults.py reference.log reduced.log restart.log
"""
import json
import sys
from pathlib import Path


def read(path):
    return json.loads(next(line[7:] for line in Path(path).read_text().splitlines()
                           if line.startswith("RESULT ")))


def compare(left, right):
    if isinstance(left, dict):
        assert left.keys() == right.keys()
        return max((compare(left[key], right[key]) for key in left), default=0.0)
    if isinstance(left, list):
        assert len(left) == len(right)
        return max((compare(a, b) for a, b in zip(left, right)), default=0.0)
    if isinstance(left, float):
        return abs(left - right)
    assert left == right, (left, right)
    return 0.0


if __name__ == "__main__":
    reference = read(sys.argv[1])["answers"]
    for path in sys.argv[2:]:
        delta = compare(reference, read(path)["answers"])
        assert delta <= 1e-5, delta
        print("%s: categorical parity passed; max numerical delta %.9g" % (path, delta))