Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

18.5 KB
Raw Blame
"""Synthetic inputs for the model-free calibration probe, `routing/calibration_probe.json`.

`dump_routing_golden.build_calibration_probe` feeds each payload to Python's own
`laya.calibrate.apply_calibration_payload` and each lookup to `laya.calibrate.apply_binning_map`, and records
what was installed, or the exception it raised, plus every warning it emitted (text and category), so a port is
checked against the shipping code on the same inputs without loading a model. This list is SDK-neutral: it is
the shared corpus for the .NET and Rust ports, and each port keeps a verbatim copy.

A payload case is `(label, text, identity)`. `text` is the JSON document, written as text because a payload may
hold what Python's `json.load` accepts and a typed value cannot state (`NaN`, `Infinity`, an integer past
float range, a duplicate key); a port reads it with its Python-json reader. `identity` names an entry of
`IDENTITIES`: the `model_id_or_path`, `subfolder` and parsed `rl_agent_config.json` of the agent the payload is
applied to, which a version 2 file is compared against. An SDK that has no equivalent of
Python's `model_id_or_path` (the .NET and Rust SDKs have none) does not emit that comparison's warning: its tests skip the
recorded warnings whose text starts "calibration was fitted for model_id_or_path=" and assert every other one.

A lookup case is `(label, map name, bucket, confidence)` for `apply_binning_map`; the name picks an entry of `MAPS`,
which are already validated maps, so only a valid one is written. A non-finite `confidence` is written as "nan"/"inf"/"-inf".
"""
import json

# The agent a payload is applied to. `cfg` is a checkpoint config: its two temperature fields are not part of the
# identity a calibration file records.
CFG = {
    "encoder": "jhu-clsp/mmBERT-base",
    "max_len": 1024,
    "head_max_len": 192,
    "option_layout": "sequential",
    "temperature": [1.2, 1.3, 1.4],
    "temperature_by_options": {"choice:2": 1.1},
}
IDENTITIES = {
    "multilingual": {"model_id_or_path": "convaiinnovations/laya", "subfolder": "multilingual", "cfg": CFG},
    "english": {"model_id_or_path": "convaiinnovations/laya", "subfolder": None, "cfg": CFG},
    "local": {"model_id_or_path": "onnx/multilingual", "subfolder": None, "cfg": CFG},
}

# The config a version 2 file records: the checkpoint config without the two temperature fields.
RECORDED_CFG = {k: v for k, v in CFG.items() if k not in ("temperature", "temperature_by_options")}

T3 = [1.5, 2.0, 1.25]
MAP4 = {"bins": 4, "values": [0.1, 0.4, 0.6, 0.9]}


def _p(label, payload, identity="multilingual"):
    return (label, json.dumps(payload), identity)


def _raw(label, text, identity="multilingual"):
    return (label, text, identity)


def _v2(**extra):
    """A version 2 payload recorded for the multilingual checkpoint, with `extra` replacing or adding fields."""
    body = {"version": 2, "temperature": T3, "temperature_by_options": {"choice:3-5": 2.5},
            "model_id_or_path": "convaiinnovations/laya", "subfolder": "multilingual", "config": RECORDED_CFG}
    body.update(extra)
    return body


def _bin(entry, other=None):
    body = {"temperature": T3, "binning_map": {"choice:3-5": entry}}
    if other is not None:
        body["binning_map"]["noul:2"] = other
    return body


PAYLOAD_CASES = [
    # --- accepted ------------------------------------------------------------------------------
    _p("temperatures only", {"temperature": T3}),
    _p("temperatures and buckets", {"temperature": T3, "temperature_by_options": {"choice:3-5": 2.5, "noul:2": 0.8}}),
    _p("null buckets", {"temperature": T3, "temperature_by_options": None}),
    _p("empty buckets", {"temperature": T3, "temperature_by_options": {}}),
    _p("bucket names are not validated", {"temperature": T3, "temperature_by_options": {"bogus": 2.0, "": 3.0}}),
    _p("integers are numbers", {"temperature": [2, 3, 1], "temperature_by_options": {"noul:2": 2}}),
    _p("version 1", {"version": 1, "temperature": T3}),
    _p("null version is 1", {"version": None, "temperature": T3}),
    _p("a binning map", {"temperature": T3, "binning_map": {
        "choice:3-5": MAP4, "noul:2": {"bins": 2, "values": [0.2, 0.8]}, "score:2": {"bins": 1, "values": [0.5]}}}),
    _p("null binning map clears", {"temperature": T3, "binning_map": None}),
    _p("empty binning map is kept", {"temperature": T3, "binning_map": {}}),
    _p("values at the bounds", _bin({"bins": 3, "values": [0.0, 1.0, 0.5]})),
    _p("integer values", _bin({"bins": 2, "values": [0, 1]})),
    _p("a bucket name is not validated", {"temperature": T3, "binning_map": {"nonsense": MAP4}}),
    _p("a quote in a bucket name", {"temperature": T3, "binning_map": {"it's": MAP4}}),
    # --- clamping and its warning ---------------------------------------------------------------
    _p("clamp high and low", {"temperature": [9.0, 0.1, 1.0]}),
    _p("clamp boundaries are accepted", {"temperature": [0.5, 5.0, 1.0]}),
    _p("clamp a bucket", {"temperature": T3, "temperature_by_options": {"choice:3-5": 99.0, "noul:2": 0.01, "score:3-5": 2.0}}),
    _p("clamp temperatures and buckets in one warning",
       {"temperature": [9.0, 1.0, 0.0], "temperature_by_options": {"choice:2": -1.0, "noul:2": 7.5}}),
    _p("negative zero", {"temperature": [-0.0, 1.0, 1.0]}),
    _raw("non-finite temperatures", '{"temperature": [NaN, 1.0, Infinity], "temperature_by_options": {"noul:2": -Infinity}}'),
    _p("a bool is not a temperature", {"temperature": [True, 1.5, False]}),
    _p("a numeric string is read as float() reads it", {"temperature": ["2.5", " 1e1 ", "abc"]}),
    _p("null and nested values are 1.0", {"temperature": [None, [1.0], {"a": 1.0}]}),
    _p("bucket values that are not numbers",
       {"temperature": T3, "temperature_by_options": {"choice:2": True, "noul:2": None, "score:2": "2.5", "choice:6-10": 7}}),
    _p("a quote in a clamped bucket name", {"temperature": T3, "temperature_by_options": {"it's": 99.0}}),
    # --- version and the checkpoint it records ---------------------------------------------------
    _p("version 2 for this checkpoint", _v2()),
    _p("version 2 written as a float", _v2(version=2.0)),
    _p("version 2 as a string", _v2(version="2")),
    _p("version 2 as a padded string", _v2(version=" 2 ")),
    _p("version 2 as a signed string", _v2(version="+2")),
    _p("version 2 with an underscore", _v2(version="0_2")),
    _p("version 2 in fullwidth digits", _v2(version="2")),
    _p("version 3 is checked too", _v2(version=3)),
    _p("a huge version is checked", _v2(version=10 ** 40)),
    _p("version 1 is never checked", _v2(version=1, model_id_or_path="elsewhere")),
    _p("version 0 is never checked", _v2(version=0, model_id_or_path="elsewhere")),
    _p("negative version is never checked", _v2(version=-3, model_id_or_path="elsewhere")),
    _p("version 2 for another model", _v2(model_id_or_path="convaiinnovations/laya-multilingual")),
    _p("version 2 for another subfolder", _v2(subfolder="typed-decisions")),
    _p("version 2 recorded without a subfolder", _v2(subfolder=None)),
    _p("version 2 with another config", _v2(config=dict(RECORDED_CFG, max_len=512))),
    _p("version 2 with an extra config key", _v2(config=dict(RECORDED_CFG, extra=1))),
    _p("version 2 with a missing config key", _v2(config={k: v for k, v in RECORDED_CFG.items() if k != "encoder"})),
    _p("version 2 with a float where the config has an integer", _v2(config=dict(RECORDED_CFG, max_len=1024.0))),
    _p("version 2 with a config that is a list", _v2(config=[1, 2])),
    _p("version 2 with no config", {k: v for k, v in _v2().items() if k != "config"}),
    _p("version 2 with a null config", _v2(config=None)),
    _p("version 2 with nothing recorded", {"version": 2, "temperature": T3}),
    _p("version 2 with a model that is not a string", _v2(model_id_or_path=7)),
    _p("version 2 with a model that is a list", _v2(model_id_or_path=["a", "b"], subfolder={"k": 1.5})),
    _p("version 2 with a quote in the model", _v2(model_id_or_path="it's")),
    _p("version 2 for the English checkpoint", _v2(subfolder=None), "english"),
    _p("version 2 for the English checkpoint recorded with a subfolder", _v2(), "english"),
    _p("version 2 for a local directory", _v2(model_id_or_path="onnx/multilingual", subfolder=None), "local"),
    _p("version 2 for a local directory recorded for the hub", _v2(), "local"),
    _p("an identity warning and a clamp warning", _v2(temperature=[9.0, 1.0, 1.0], model_id_or_path="x")),
    # --- refusals: the payload ------------------------------------------------------------------
    _p("payload is a list", [1, 2, 3]),
    _p("payload is a number", 3),
    _p("payload is a float", 1.5),
    _p("payload is a string", "abc"),
    _p("payload is null", None),
    _p("payload is true", True),
    _p("payload is empty", {}),
    # --- refusals: version ----------------------------------------------------------------------
    _p("version is a bool", {"version": True, "temperature": T3}),
    _p("version is false", {"version": False, "temperature": T3}),
    _p("version is a list", {"version": [2], "temperature": T3}),
    _p("version is an object", {"version": {"v": 2}, "temperature": T3}),
    _p("version is not whole", {"version": 1.9, "temperature": T3}),
    _p("version is not whole and large", {"version": 2.5, "temperature": T3}),
    _p("version is a word", {"version": "abc", "temperature": T3}),
    _p("version is an empty string", {"version": "", "temperature": T3}),
    _p("version is a decimal string", {"version": "1.5", "temperature": T3}),
    _p("version is a double underscore string", {"version": "1__2", "temperature": T3}),
    _p("version has a quote", {"version": "it's", "temperature": T3}),
    _raw("version is NaN", '{"version": NaN, "temperature": [1.0, 1.0, 1.0]}'),
    _raw("version is Infinity", '{"version": Infinity, "temperature": [1.0, 1.0, 1.0]}'),
    _raw("version is an overflowing float", '{"version": 1e999, "temperature": [1.0, 1.0, 1.0]}'),
    _raw("version is a huge whole float", '{"version": 1e300, "temperature": [1.0, 1.0, 1.0]}'),
    # --- refusals: temperatures -----------------------------------------------------------------
    _p("temperature is missing", {"version": 1}),
    _p("temperature is null", {"temperature": None}),
    _p("temperature has two values", {"temperature": [1.0, 2.0]}),
    _p("temperature has four values", {"temperature": [1.0, 2.0, 3.0, 4.0]}),
    _p("temperature is empty", {"temperature": []}),
    _p("temperature is a number", {"temperature": 2.0}),
    _p("temperature is a string of three", {"temperature": "abc"}),
    _p("temperature is an object of three", {"temperature": {"a": 1, "b": 2, "c": 3}}),
    _p("buckets are a list", {"temperature": T3, "temperature_by_options": [1.0]}),
    _p("buckets are a number", {"temperature": T3, "temperature_by_options": 3}),
    _p("buckets are a float", {"temperature": T3, "temperature_by_options": 1.5}),
    _p("buckets are a string", {"temperature": T3, "temperature_by_options": "x"}),
    _p("buckets are false", {"temperature": T3, "temperature_by_options": False}),
    _p("buckets are an empty list", {"temperature": T3, "temperature_by_options": []}),
    # --- refusals: the binning map --------------------------------------------------------------
    _p("binning map is a list", {"temperature": T3, "binning_map": []}),
    _p("binning map is a number", {"temperature": T3, "binning_map": 3}),
    _p("binning map is a string", {"temperature": T3, "binning_map": "x"}),
    _p("binning map is false", {"temperature": T3, "binning_map": False}),
    _p("entry is a list", {"temperature": T3, "binning_map": {"choice:3-5": [0.1, 0.9]}}),
    _p("entry is null", {"temperature": T3, "binning_map": {"choice:3-5": None}}),
    _p("entry is a number", {"temperature": T3, "binning_map": {"choice:3-5": 0.5}}),
    _p("entry is a string", {"temperature": T3, "binning_map": {"choice:3-5": "x"}}),
    _p("entry is empty", {"temperature": T3, "binning_map": {"choice:3-5": {}}}),
    _p("entry is true", {"temperature": T3, "binning_map": {"choice:3-5": True}}),
    _p("a quote in a refused bucket name", {"temperature": T3, "binning_map": {"it's": 3}}),
    _p("bins is missing", _bin({"values": [0.1, 0.9]})),
    _p("bins is null", _bin({"bins": None, "values": [0.1, 0.9]})),
    _p("bins is zero", _bin({"bins": 0, "values": []})),
    _p("bins is negative", _bin({"bins": -2, "values": [0.1, 0.9]})),
    _p("bins is a whole float", _bin({"bins": 2.0, "values": [0.1, 0.9]})),
    _p("bins is a float", _bin({"bins": 2.5, "values": [0.1, 0.9]})),
    _p("bins is a string", _bin({"bins": "2", "values": [0.1, 0.9]})),
    _p("bins is true", _bin({"bins": True, "values": [0.1]})),
    _p("bins is a list", _bin({"bins": [2], "values": [0.1, 0.9]})),
    _p("bins is a huge integer", _bin({"bins": 10 ** 30, "values": [0.1, 0.9]})),
    _p("bins is a negative huge integer", _bin({"bins": -10 ** 30, "values": [0.1, 0.9]})),
    _raw("bins is NaN", '{"temperature": [1.0, 1.0, 1.0], "binning_map": {"a": {"bins": NaN, "values": [0.5]}}}'),
    _p("values is missing", _bin({"bins": 2})),
    _p("values is null", _bin({"bins": 2, "values": None})),
    _p("values is too short", _bin({"bins": 3, "values": [0.1, 0.9]})),
    _p("values is too long", _bin({"bins": 1, "values": [0.1, 0.9]})),
    _p("values is a string", _bin({"bins": 3, "values": "abc"})),
    _p("values is an object of the right size", _bin({"bins": 2, "values": {"a": 0.1, "b": 0.9}})),
    _p("values is a number", _bin({"bins": 1, "values": 0.5})),
    _p("a value is null", _bin({"bins": 2, "values": [0.1, None]})),
    _p("a value is a string", _bin({"bins": 2, "values": [0.1, "0.9"]})),
    _p("a value is a word", _bin({"bins": 2, "values": [0.1, "abc"]})),
    _p("a value is true", _bin({"bins": 2, "values": [0.1, True]})),
    _p("a value is false", _bin({"bins": 2, "values": [False, 0.5]})),
    _p("a value is a list", _bin({"bins": 2, "values": [0.1, [0.9]]})),
    _p("a value is an object", _bin({"bins": 2, "values": [{"v": 0.1}, 0.5]})),
    _raw("a value is NaN", '{"temperature": [1.0, 1.0, 1.0], "binning_map": {"a": {"bins": 2, "values": [0.1, NaN]}}}'),
    _raw("a value is Infinity",
         '{"temperature": [1.0, 1.0, 1.0], "binning_map": {"a": {"bins": 2, "values": [0.1, Infinity]}}}'),
    _raw("a value is negative Infinity",
         '{"temperature": [1.0, 1.0, 1.0], "binning_map": {"a": {"bins": 1, "values": [-Infinity]}}}'),
    _raw("a value overflows to Infinity",
         '{"temperature": [1.0, 1.0, 1.0], "binning_map": {"a": {"bins": 1, "values": [1e999]}}}'),
    _p("a value is below zero", _bin({"bins": 2, "values": [-0.1, 0.5]})),
    _p("a value is above one", _bin({"bins": 2, "values": [0.5, 1.0000001]})),
    _p("a value is an integer above one", _bin({"bins": 2, "values": [0.5, 2]})),
    _p("a value is -0.0", _bin({"bins": 1, "values": [-0.0]})),
    _p("a value is a huge integer", _bin({"bins": 2, "values": [0.5, 10 ** 400]})),
    _p("a huge integer value after a bad one", _bin({"bins": 3, "values": [None, 10 ** 400, 0.5]})),
    _p("a bad value after a huge integer", _bin({"bins": 3, "values": [10 ** 400, None, 0.5]})),
    _p("a string value before a list value", _bin({"bins": 2, "values": ["0.5", [1]]})),
    _p("an out-of-range value before a finite one", _bin({"bins": 2, "values": [2.0, 0.5]})),
    _raw("an out-of-range value before a non-finite one",
         '{"temperature": [1.0, 1.0, 1.0], "binning_map": {"a": {"bins": 2, "values": [2.0, NaN]}}}'),
    _p("the second entry is the bad one", _bin(MAP4, {"bins": 2, "values": [0.1, 5.0]})),
    _p("the first entry is the bad one", _bin({"bins": 2, "values": [0.1, 5.0]}, {"bins": 2, "values": [0.2, "x"]})),
    _p("a refusal beats a temperature warning",
       {"temperature": [9.0, 1.0, 1.0], "binning_map": {"a": {"bins": 1, "values": [3.0]}}}),
    _p("a refusal beats an identity warning", _v2(binning_map={"a": {"bins": 1, "values": [3.0]}})),
    _raw("a duplicate key keeps its last value",
         '{"temperature": [1.0, 2.0, 3.0], "temperature": [4.0, 5.0, 1.5], "binning_map": {"a": {"bins": 1, "values": [0.5]}, '
         '"a": {"bins": 2, "values": [0.1, 0.9]}}}'),
    # --- an integer too large for a float -------------------------------------------------------
    _p("a temperature that is a huge integer", {"temperature": [10 ** 400, 1.0, 1.0]}),
    _p("a bucket temperature that is a huge integer", {"temperature": T3, "temperature_by_options": {"noul:2": 10 ** 400}}),
    _p("a huge integer after an identity warning", _v2(temperature=[10 ** 400, 1.0, 1.0], model_id_or_path="x")),
]

BIN4 = {"bins": 4, "values": [0.1, 0.4, 0.6, 0.9]}
BIN1 = {"bins": 1, "values": [0.5]}
BIN10 = {"bins": 10, "values": [round(i / 10 + 0.05, 2) for i in range(10)]}
BIN100 = {"bins": 100, "values": [round((i * 37 % 100) / 100, 2) for i in range(100)]}
MAP = {"choice:3-5": BIN4, "noul:2": {"bins": 2, "values": [0.25, 0.75]}, "score:2": BIN1, "choice:6-10": BIN10,
       "choice:11+": BIN100}

MAPS = {"main": MAP, "empty": {}}

LOOKUP_CASES = (
    # bin edges and the ends
    [("choice:3-5 at %r" % c, "main", "choice:3-5", c)
     for c in (0.0, 1e-300, 0.1, 0.2499999999999999, 0.25, 0.2500000000000001, 0.3, 0.4999999999999999, 0.5,
               0.5000000000000001, 0.6, 0.7499999999999999, 0.75, 0.7500000000000001, 0.9, 0.9999999999999999, 1.0)]
    # outside [0, 1]: clipped by the bin index, not rejected
    + [("choice:3-5 at %r" % c, "main", "choice:3-5", c) for c in (-0.0, -0.1, -1e300, 1.0000000000000002, 1.5, 7.0, 1e300)]
    # one bin takes everything
    + [("one bin at %r" % c, "main", "score:2", c) for c in (0.0, 0.5, 1.0)]
    # a bucket the map was not fit for passes through, with whatever the caller holds
    + [("miss at %r" % c, "main", "score:3-5", c) for c in (0.0, 0.123456789, 0.5, 1.0, 1.5, -0.25)]
    + [("miss on an empty bucket name", "main", "", 0.4), ("miss on a nearly right name", "main", "choice:3-5 ", 0.4),
       ("miss on another case", "main", "CHOICE:3-5", 0.4), ("empty map", "empty", "choice:3-5", 0.4)]
    # ten bins: float products that fall just under a whole number are truncated, not rounded
    + [("ten bins at %r" % c, "main", "choice:6-10", c) for c in (0.1, 0.2, 0.3, 0.7, 0.57, 0.9, 0.95, 0.99, 0.999999)]
    + [("hundred bins at %r" % c, "main", "choice:11+", c) for c in (0.01, 0.07, 0.29, 0.57, 0.58, 0.99, 1.0)]
    + [("two bins at %r" % c, "main", "noul:2", c) for c in (0.5, 0.7499999999999999, 0.75, 1.0)]
    # a value float() takes and int() refuses
    + [("nan", "main", "choice:3-5", float("nan")), ("inf", "main", "choice:3-5", float("inf")),
       ("-inf", "main", "choice:3-5", float("-inf")), ("nan on a miss", "main", "score:3-5", float("nan")),
       ("inf on a miss", "main", "score:3-5", float("inf"))]
)