Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(fast): run the TileLang fast path in the agent's autocast dtype (bf16 or fp16)

cklxx96c5a6db3328
diff --git a/BENCHMARKS.md b/BENCHMARKS.md
index 8b8dd9b0..10814752 100644
--- a/BENCHMARKS.md
+++ b/BENCHMARKS.md
@@ -304,6 +304,34 @@ fast path is (0.0446 against 0.0455), so this table does not show that the fast
Dataset accuracy / ECE (AG News, dair-ai emotion, 1,000 samples each) are identical within noise; see
`benchmarks/bench_fast.py --eval 1000`.
+### fp16
+
+The fast path runs in the agent's autocast dtype when `accelerate()` is called, so an agent set to fp16
+(`agent.dtype = torch.float16`, or the CUDA autocast override proposed for #443) gets fp16 kernels and fp16
+weights; the residual stream and every accumulation stay fp32 in both dtypes. Same fixed set,
+`parity_fast.py --dtype fp16 | bf16`, RTX 4070 Ti SUPER; per-option probabilities in
+`benchmarks/results/parity_*_rtx4070.json` (the bf16 columns are the table above, plus a bf16 run of
+`laya-typed-decisions`):
+
+| checkpoint | type | n | max \|p_fast - p_fp32\| bf16 | max \|p_fast - p_fp32\| fp16 | argmax fast = fp32, bf16 | fp16 |
+|---|---|---|---|---|---|---|
+| laya | choice | 48 | 0.022 | **0.004** | 47/48 | 48/48 |
+| laya | noul | 180 | 0.043 | **0.005** | 180/180 | 180/180 |
+| laya | score | 60 | 0.011 | **0.003** | 60/60 | 60/60 |
+| laya-multilingual | choice | 48 | 0.015 | **0.002** | 47/48 | 48/48 |
+| laya-multilingual | noul | 180 | 0.045 | **0.009** | 179/180 | 180/180 |
+| laya-multilingual | score | 60 | 0.009 | **0.001** | 59/60 | 60/60 |
+| laya-typed-decisions | choice | 48 | 0.019 | **0.002** | 47/48 | 47/48 |
+| laya-typed-decisions | noul | 180 | 0.023 | **0.005** | 180/180 | 180/180 |
+| laya-typed-decisions | score | 60 | 0.009 | **0.001** | 60/60 | 60/60 |
+
+In fp16 the fast path is 3-10x closer to fp32 than in bf16 and agrees with the fp16 stock path on every argmax
+(864/864; the most it moves a probability against fp16 stock is 0.009). The one fp16 disagreement with fp32 is a
+`laya-typed-decisions` choice question whose top two options are 0.001 apart in fp32; the fp16 stock path flips it too.
+`agent.predict()` latency shows no consistent difference between the dtypes: on every case of the table below, on
+both checkpoints, fp16 and bf16 are within 10% of each other in both directions (single runs of 50 iterations), for
+stock and fast alike.
+
### Latency, `agent.predict()` end to end (ms, incl. tokenization)
| checkpoint | case | stock | fast | speedup |
diff --git a/README.md b/README.md
index 0560d3e6..6fedc38a 100644
--- a/README.md
+++ b/README.md
@@ -565,6 +565,8 @@ agent.predict(state, questions) # same API, same answ
Numerics: on a fixed set of 60 states the fast path stays within 0.046 of an fp32 forward and within 0.076 of the stock
bf16 path (max |Δp| ≤ 0.05 vs fp32 on both checkpoints, argmax agreement ≥ 47/48 per question type; every per-option
probability is in `benchmarks/results/parity_*.json`) — see `benchmarks/parity_fast.py` and [BENCHMARKS.md](BENCHMARKS.md#gpu-fast-path).
+The fast path runs in the agent's autocast dtype at the time `accelerate()` is called: bf16 by default, fp16 if
+`agent.dtype` is `torch.float16`, where it stays within 0.009 of fp32 on the same set ([BENCHMARKS.md](BENCHMARKS.md#fp16)).
Falls back to the stock forward on CPU/MPS or when `tilelang` is not installed; `agent.deaccelerate()`
restores it. Kernels compile once per shape bucket on first use (a few seconds, cached on disk).
diff --git a/benchmarks/bench_fast.py b/benchmarks/bench_fast.py
index d69b3c10..4ae048df 100644
--- a/benchmarks/bench_fast.py
+++ b/benchmarks/bench_fast.py
@@ -17,12 +17,16 @@ ap = argparse.ArgumentParser()
ap.add_argument("--model", default="convaiinnovations/laya"); ap.add_argument("--subfolder", default=None)
ap.add_argument("--eval", type=int, default=0, help="samples per dataset for the accuracy comparison (0 = skip)")
ap.add_argument("--iters", type=int, default=30); ap.add_argument("--json", default=None)
+ap.add_argument("--dtype", choices=["bf16", "fp16"], default=None, help="autocast dtype for stock and fast (default: the agent's)")
args = ap.parse_args()
agent = laya.load(args.model, subfolder=args.subfolder)
if agent.device.type != "cuda":
sys.exit("needs a CUDA device")
-report = {"model": args.model, "subfolder": args.subfolder, "gpu": torch.cuda.get_device_name(0), "torch": torch.__version__}
+if args.dtype:
+ agent.dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[args.dtype]
+report = {"model": args.model, "subfolder": args.subfolder, "gpu": torch.cuda.get_device_name(0), "torch": torch.__version__,
+ "dtype": str(agent.dtype)}
Q = {"department": {"type": "choice", "instructions": "Which team should handle this?",
"criteria": {"billing": "invoices, refunds", "technical": "bugs, outages", "sales": "pricing", "shipping": "delivery"}},
diff --git a/benchmarks/parity_fast.py b/benchmarks/parity_fast.py
index 95d300c4..ef36b502 100644
--- a/benchmarks/parity_fast.py
+++ b/benchmarks/parity_fast.py
@@ -1,7 +1,10 @@
"""Parity evidence for the TileLang fast path: a fixed, deterministic set of states x questions, answered by the
-stock forward (bf16 autocast) and by the fast path, with an fp32 forward as the reference.
+stock forward (16-bit autocast) and by the fast path, with an fp32 forward as the reference.
- python benchmarks/parity_fast.py [--subfolder multilingual] [--json benchmarks/results/parity_<name>.json]
+ python benchmarks/parity_fast.py [--subfolder multilingual] [--dtype bf16|fp16] [--json benchmarks/results/parity_<name>.json]
+
+`--dtype` sets the autocast dtype of both the stock path and the fast path (default: the agent's own, bf16 for
+the shipped checkpoints on compute capability >= 8).
Writes every per-option probability from all three paths so the comparison can be re-checked without a GPU, and
prints the summary the PR quotes: max |p_fast - p_stock|, max |p_* - p_fp32|, argmax agreement, per question type.
@@ -63,28 +66,34 @@ def probs(agent, b, meta, amp):
def main():
ap = argparse.ArgumentParser(); ap.add_argument("--model", default="convaiinnovations/laya"); ap.add_argument("--subfolder", default=None); ap.add_argument("--json", default=None)
+ ap.add_argument("--dtype", choices=["bf16", "fp16"], default=None, help="autocast dtype for stock and fast (default: the agent's)")
a = ap.parse_args()
agent = laya.load(a.model, subfolder=a.subfolder)
if agent.device.type != "cuda":
sys.exit("needs CUDA")
+ if a.dtype:
+ agent.dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[a.dtype]
+ tag = {torch.bfloat16: "bf16", torch.float16: "fp16"}[agent.dtype]
+ stock_key = "p_stock_" + tag
cases = []
for name, st, qs in states():
b, meta = batch(agent, st, qs)
agent.deaccelerate(); p32 = probs(agent, b, meta, amp=False); pst = probs(agent, b, meta, amp=True)
assert agent.accelerate(strict=True); pfa = probs(agent, b, meta, amp=True)
for (qid, t, k), x, y, z in zip(meta, p32, pst, pfa):
- cases.append({"state": name, "question": qid, "type": t, "k": k, "p_fp32": x, "p_stock_bf16": y, "p_fast": z})
+ cases.append({"state": name, "question": qid, "type": t, "k": k, "p_fp32": x, stock_key: y, "p_fast": z})
by_t = {}
for c in cases:
- d = by_t.setdefault(c["type"], {"n": 0, "d_fast_stock": 0.0, "d_fast_fp32": 0.0, "d_stock_fp32": 0.0, "agree_fast_stock": 0, "agree_fast_fp32": 0})
- x, y, z = map(np.array, (c["p_fp32"], c["p_stock_bf16"], c["p_fast"]))
+ d = by_t.setdefault(c["type"], {"n": 0, "d_fast_stock": 0.0, "d_fast_fp32": 0.0, "d_stock_fp32": 0.0, "agree_fast_stock": 0, "agree_fast_fp32": 0, "agree_stock_fp32": 0})
+ x, y, z = map(np.array, (c["p_fp32"], c[stock_key], c["p_fast"]))
d["n"] += 1; d["d_fast_stock"] = max(d["d_fast_stock"], float(abs(z - y).max())); d["d_fast_fp32"] = max(d["d_fast_fp32"], float(abs(z - x).max()))
d["d_stock_fp32"] = max(d["d_stock_fp32"], float(abs(y - x).max())); d["agree_fast_stock"] += int(z.argmax() == y.argmax()); d["agree_fast_fp32"] += int(z.argmax() == x.argmax())
+ d["agree_stock_fp32"] += int(y.argmax() == x.argmax())
gpu = torch.cuda.get_device_name(0)
print(f"\n{a.model}/{a.subfolder or ''} {gpu} torch {torch.__version__} dtype {agent.dtype}\n{len(cases)} questions over {len(set(c['state'] for c in cases))} fixed states")
- print(f"{'type':8s} {'n':>4s} {'max|fast-stock|':>16s} {'max|fast-fp32|':>15s} {'max|stock-fp32|':>16s} {'argmax fast=stock':>18s} {'fast=fp32':>10s}")
+ print(f"{'type':8s} {'n':>4s} {'max|fast-stock|':>16s} {'max|fast-fp32|':>15s} {'max|stock-fp32|':>16s} {'argmax fast=stock':>18s} {'fast=fp32':>10s} {'stock=fp32':>11s}")
for t, d in sorted(by_t.items()):
- print(f"{t:8s} {d['n']:4d} {d['d_fast_stock']:16.4f} {d['d_fast_fp32']:15.4f} {d['d_stock_fp32']:16.4f} {d['agree_fast_stock']:>13d}/{d['n']:<4d} {d['agree_fast_fp32']:>6d}/{d['n']}")
+ print(f"{t:8s} {d['n']:4d} {d['d_fast_stock']:16.4f} {d['d_fast_fp32']:15.4f} {d['d_stock_fp32']:16.4f} {d['agree_fast_stock']:>13d}/{d['n']:<4d} {d['agree_fast_fp32']:>6d}/{d['n']:<4d} {d['agree_stock_fp32']:>6d}/{d['n']}")
if a.json:
os.makedirs(os.path.dirname(a.json), exist_ok=True)
json.dump({"model": a.model, "subfolder": a.subfolder, "gpu": gpu, "torch": torch.__version__, "dtype": str(agent.dtype), "summary": by_t, "cases": cases}, open(a.json, "w"), indent=1)
diff --git a/benchmarks/results/parity_english_fp16_rtx4070.json b/benchmarks/results/parity_english_fp16_rtx4070.json
new file mode 100644
index 00000000..3c691dda
--- /dev/null
+++ b/benchmarks/results/parity_english_fp16_rtx4070.json
@@ -0,0 +1,6122 @@
+{
+ "model": "convaiinnovations/laya",
+ "subfolder": null,
+ "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER",
+ "torch": "2.11.0+cu130",
+ "dtype": "torch.float16",
+ "summary": {
+ "choice": {
+ "n": 48,
+ "d_fast_stock": 0.006913185119628906,
+ "d_fast_fp32": 0.004358470439910889,
+ "d_stock_fp32": 0.005166053771972656,
+ "agree_fast_stock": 48,
+ "agree_fast_fp32": 48,
+ "agree_stock_fp32": 48
+ },
+ "noul": {
+ "n": 180,
+ "d_fast_stock": 0.00919044017791748,
+ "d_fast_fp32": 0.005239367485046387,
+ "d_stock_fp32": 0.013456881046295166,
+ "agree_fast_stock": 180,
+ "agree_fast_fp32": 180,
+ "agree_stock_fp32": 180
+ },
+ "score": {
+ "n": 60,
+ "d_fast_stock": 0.004023194313049316,
+ "d_fast_fp32": 0.003058493137359619,
+ "d_stock_fp32": 0.003620922565460205,
+ "agree_fast_stock": 60,
+ "agree_fast_fp32": 60,
+ "agree_stock_fp32": 60
+ }
+ },
+ "cases": [
+ {
+ "state": "triage/0",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9947391152381897,
+ 0.00011028684821212664,
+ 0.003915545996278524,
+ 5.80661217099987e-05,
+ 0.000818361178971827,
+ 0.00035863046650774777
+ ],
+ "p_stock_fp16": [
+ 0.9947934746742249,
+ 0.00010983312677126378,
+ 0.0038698615971952677,
+ 5.7878001825883985e-05,
+ 0.0008119595586322248,
+ 0.00035697765997610986
+ ],
+ "p_fast": [
+ 0.9947484135627747,
+ 0.00011025802814401686,
+ 0.003903847187757492,
+ 5.810189759358764e-05,
+ 0.000820291053969413,
+ 0.00035905916593037546
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8415257334709167,
+ 0.15847428143024445
+ ],
+ "p_stock_fp16": [
+ 0.8382167816162109,
+ 0.16178321838378906
+ ],
+ "p_fast": [
+ 0.8415002226829529,
+ 0.15849970281124115
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.024894164875149727,
+ 0.23411326110363007,
+ 0.6823216676712036,
+ 0.058670949190855026
+ ],
+ "p_stock_fp16": [
+ 0.02483276277780533,
+ 0.23377317190170288,
+ 0.6824166178703308,
+ 0.058977384120225906
+ ],
+ "p_fast": [
+ 0.024807404726743698,
+ 0.23307879269123077,
+ 0.6830525994300842,
+ 0.05906118080019951
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.07681716233491898,
+ 0.9231828451156616
+ ],
+ "p_stock_fp16": [
+ 0.07668380439281464,
+ 0.9233161807060242
+ ],
+ "p_fast": [
+ 0.0770302340388298,
+ 0.9229697585105896
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.046758126467466354,
+ 0.9532419443130493
+ ],
+ "p_stock_fp16": [
+ 0.046594638377428055,
+ 0.9534053802490234
+ ],
+ "p_fast": [
+ 0.046594638377428055,
+ 0.9534053802490234
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.002989979926496744,
+ 0.9817689657211304,
+ 0.0013363086618483067,
+ 0.0015498370630666614,
+ 0.001197210163809359,
+ 0.011157740838825703
+ ],
+ "p_stock_fp16": [
+ 0.003004832426086068,
+ 0.9816985130310059,
+ 0.0013359930599108338,
+ 0.0015528089134022593,
+ 0.001199917634949088,
+ 0.01120800245553255
+ ],
+ "p_fast": [
+ 0.002976063871756196,
+ 0.9818413257598877,
+ 0.0013309781206771731,
+ 0.0015469801146537066,
+ 0.0011930810287594795,
+ 0.011111541651189327
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.4236339330673218,
+ 0.5763660669326782
+ ],
+ "p_stock_fp16": [
+ 0.4201236665248871,
+ 0.5798763036727905
+ ],
+ "p_fast": [
+ 0.42036163806915283,
+ 0.5796384215354919
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.1028120368719101,
+ 0.3615639805793762,
+ 0.47248536348342896,
+ 0.06313859671354294
+ ],
+ "p_stock_fp16": [
+ 0.10269985347986221,
+ 0.3614455461502075,
+ 0.4727969765663147,
+ 0.06305757164955139
+ ],
+ "p_fast": [
+ 0.1028720885515213,
+ 0.36169835925102234,
+ 0.4722045063972473,
+ 0.06322504580020905
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9656645059585571,
+ 0.0343354307115078
+ ],
+ "p_stock_fp16": [
+ 0.9657283425331116,
+ 0.0342717170715332
+ ],
+ "p_fast": [
+ 0.9655506014823914,
+ 0.03444942831993103
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9905862212181091,
+ 0.009413759224116802
+ ],
+ "p_stock_fp16": [
+ 0.9905577898025513,
+ 0.009442228823900223
+ ],
+ "p_fast": [
+ 0.990511953830719,
+ 0.009488007053732872
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.15150891244411469,
+ 0.026228059083223343,
+ 0.09237370640039444,
+ 0.16930243372917175,
+ 0.02988293580710888,
+ 0.5307039618492126
+ ],
+ "p_stock_fp16": [
+ 0.1508682370185852,
+ 0.026165809482336044,
+ 0.09195410460233688,
+ 0.16871707141399384,
+ 0.029765795916318893,
+ 0.5325289368629456
+ ],
+ "p_fast": [
+ 0.1511802077293396,
+ 0.02618793211877346,
+ 0.09198691695928574,
+ 0.16937580704689026,
+ 0.029849199578166008,
+ 0.5314199328422546
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9612442851066589,
+ 0.03875567764043808
+ ],
+ "p_stock_fp16": [
+ 0.9613845348358154,
+ 0.03861546143889427
+ ],
+ "p_fast": [
+ 0.9613573551177979,
+ 0.03864266350865364
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.477916419506073,
+ 0.39026403427124023,
+ 0.11922533065080643,
+ 0.012594283558428288
+ ],
+ "p_stock_fp16": [
+ 0.4769713580608368,
+ 0.390816330909729,
+ 0.11960034817457199,
+ 0.012611939571797848
+ ],
+ "p_fast": [
+ 0.4772091209888458,
+ 0.39062947034835815,
+ 0.11954315751791,
+ 0.012618225999176502
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9946468472480774,
+ 0.005353105254471302
+ ],
+ "p_stock_fp16": [
+ 0.9946593642234802,
+ 0.005340661853551865
+ ],
+ "p_fast": [
+ 0.9946281313896179,
+ 0.005371877457946539
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9833458662033081,
+ 0.016654079779982567
+ ],
+ "p_stock_fp16": [
+ 0.9834452271461487,
+ 0.016554752364754677
+ ],
+ "p_fast": [
+ 0.9834451079368591,
+ 0.016554875299334526
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.010802879929542542,
+ 0.048171281814575195,
+ 0.0042482661083340645,
+ 0.012916400097310543,
+ 0.04330618679523468,
+ 0.8805549740791321
+ ],
+ "p_stock_fp16": [
+ 0.010772710666060448,
+ 0.048141639679670334,
+ 0.004247590433806181,
+ 0.012893245555460453,
+ 0.04358002915978432,
+ 0.8803648352622986
+ ],
+ "p_fast": [
+ 0.010856317356228828,
+ 0.04858268052339554,
+ 0.0042722043581306934,
+ 0.012942654080688953,
+ 0.0430530309677124,
+ 0.8802931308746338
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9959781169891357,
+ 0.004021893721073866
+ ],
+ "p_stock_fp16": [
+ 0.9959613680839539,
+ 0.004038591869175434
+ ],
+ "p_fast": [
+ 0.9959269165992737,
+ 0.004073108080774546
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.17910070717334747,
+ 0.1875993013381958,
+ 0.58836430311203,
+ 0.044935621321201324
+ ],
+ "p_stock_fp16": [
+ 0.17942914366722107,
+ 0.18694156408309937,
+ 0.5889003872871399,
+ 0.04472891986370087
+ ],
+ "p_fast": [
+ 0.18019548058509827,
+ 0.1870993822813034,
+ 0.5879603028297424,
+ 0.0447448305785656
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.3379279906189367e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.295078622121082e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 1.336188848455322e-08
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9940734505653381,
+ 0.005926612298935652
+ ],
+ "p_stock_fp16": [
+ 0.9940648674964905,
+ 0.005935146939009428
+ ],
+ "p_fast": [
+ 0.9940356016159058,
+ 0.005964386276900768
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.1352403461933136,
+ 0.01730111986398697,
+ 0.00795932300388813,
+ 0.010730478912591934,
+ 0.032282836735248566,
+ 0.7964859008789062
+ ],
+ "p_stock_fp16": [
+ 0.13563242554664612,
+ 0.017378972843289375,
+ 0.007972229272127151,
+ 0.010769793763756752,
+ 0.0323416143655777,
+ 0.7959049344062805
+ ],
+ "p_fast": [
+ 0.13531668484210968,
+ 0.017355455085635185,
+ 0.007977006025612354,
+ 0.010734234005212784,
+ 0.03223482891917229,
+ 0.7963817715644836
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9996178150177002,
+ 0.0003821933642029762
+ ],
+ "p_stock_fp16": [
+ 0.9996209144592285,
+ 0.00037905829958617687
+ ],
+ "p_fast": [
+ 0.9996165037155151,
+ 0.0003835248062387109
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.003483779262751341,
+ 0.008040246553719044,
+ 0.16993559896945953,
+ 0.8185403943061829
+ ],
+ "p_stock_fp16": [
+ 0.0034847573842853308,
+ 0.008039250038564205,
+ 0.17005020380020142,
+ 0.8184257745742798
+ ],
+ "p_fast": [
+ 0.0034797396510839462,
+ 0.008035518229007721,
+ 0.16963960230350494,
+ 0.8188450932502747
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9931904673576355,
+ 0.00680957268923521
+ ],
+ "p_stock_fp16": [
+ 0.9932090640068054,
+ 0.006790942046791315
+ ],
+ "p_fast": [
+ 0.9932304620742798,
+ 0.00676956819370389
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9998910427093506,
+ 0.00010890261910390109
+ ],
+ "p_stock_fp16": [
+ 0.9998911619186401,
+ 0.00010885704250540584
+ ],
+ "p_fast": [
+ 0.999891996383667,
+ 0.00010799682058859617
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.2945304214954376,
+ 0.031219109892845154,
+ 0.0233877282589674,
+ 0.06210966408252716,
+ 0.08097606897354126,
+ 0.5077769756317139
+ ],
+ "p_stock_fp16": [
+ 0.2941577434539795,
+ 0.031277697533369064,
+ 0.023311806842684746,
+ 0.06214240938425064,
+ 0.08085134625434875,
+ 0.5082590579986572
+ ],
+ "p_fast": [
+ 0.29349666833877563,
+ 0.031237896531820297,
+ 0.023373262956738472,
+ 0.0620027519762516,
+ 0.08078790456056595,
+ 0.5091015696525574
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.20980310440063477,
+ 0.7901968359947205
+ ],
+ "p_stock_fp16": [
+ 0.21044158935546875,
+ 0.7895584106445312
+ ],
+ "p_fast": [
+ 0.20971235632896423,
+ 0.7902876734733582
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2347777634859085,
+ 0.14355722069740295,
+ 0.49825823307037354,
+ 0.12340675294399261
+ ],
+ "p_stock_fp16": [
+ 0.23542186617851257,
+ 0.14461490511894226,
+ 0.4969301223754883,
+ 0.12303309142589569
+ ],
+ "p_fast": [
+ 0.2351054847240448,
+ 0.1435067355632782,
+ 0.49820461869239807,
+ 0.12318312376737595
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7551621198654175,
+ 0.2448379099369049
+ ],
+ "p_stock_fp16": [
+ 0.7572562098503113,
+ 0.24274376034736633
+ ],
+ "p_fast": [
+ 0.75743567943573,
+ 0.24256429076194763
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9279112219810486,
+ 0.07208879292011261
+ ],
+ "p_stock_fp16": [
+ 0.9272645115852356,
+ 0.07273544371128082
+ ],
+ "p_fast": [
+ 0.9280183911323547,
+ 0.07198163121938705
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.33260369300842285,
+ 0.13441450893878937,
+ 0.05236406251788139,
+ 0.015323297120630741,
+ 0.12186992913484573,
+ 0.3434244990348816
+ ],
+ "p_stock_fp16": [
+ 0.33420243859291077,
+ 0.13350392878055573,
+ 0.05277532711625099,
+ 0.015328527428209782,
+ 0.12072285264730453,
+ 0.3434668779373169
+ ],
+ "p_fast": [
+ 0.33111652731895447,
+ 0.13479721546173096,
+ 0.05282693728804588,
+ 0.015381022356450558,
+ 0.12190764397382736,
+ 0.3439706265926361
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9908686876296997,
+ 0.009131325408816338
+ ],
+ "p_stock_fp16": [
+ 0.9908862113952637,
+ 0.009113779291510582
+ ],
+ "p_fast": [
+ 0.9908667802810669,
+ 0.009133229963481426
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.1992260068655014,
+ 0.10026822984218597,
+ 0.6071023941040039,
+ 0.09340337663888931
+ ],
+ "p_stock_fp16": [
+ 0.1993650496006012,
+ 0.10076232254505157,
+ 0.6063408255577087,
+ 0.09353181719779968
+ ],
+ "p_fast": [
+ 0.20007047057151794,
+ 0.10062632709741592,
+ 0.6061140298843384,
+ 0.09318917989730835
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.003025050973519683,
+ 0.9969749450683594
+ ],
+ "p_stock_fp16": [
+ 0.003048541024327278,
+ 0.9969514608383179
+ ],
+ "p_fast": [
+ 0.0030544826295226812,
+ 0.9969455599784851
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9124882817268372,
+ 0.08751174062490463
+ ],
+ "p_stock_fp16": [
+ 0.9177091717720032,
+ 0.08229081332683563
+ ],
+ "p_fast": [
+ 0.9177276492118835,
+ 0.08227238059043884
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.015215393155813217,
+ 0.005658997688442469,
+ 0.008860105648636818,
+ 0.03527345880866051,
+ 0.004129140637814999,
+ 0.9308629035949707
+ ],
+ "p_stock_fp16": [
+ 0.015331286005675793,
+ 0.005712123122066259,
+ 0.008925210684537888,
+ 0.03568113595247269,
+ 0.0041831620037555695,
+ 0.9301671385765076
+ ],
+ "p_fast": [
+ 0.015039045363664627,
+ 0.00562517112120986,
+ 0.008789347484707832,
+ 0.035125114023685455,
+ 0.004079450387507677,
+ 0.931341826915741
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9991705417633057,
+ 0.0008294567815028131
+ ],
+ "p_stock_fp16": [
+ 0.9991740584373474,
+ 0.0008259545429609716
+ ],
+ "p_fast": [
+ 0.9991689920425415,
+ 0.0008310069679282606
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5344459414482117,
+ 0.33613914251327515,
+ 0.11765743792057037,
+ 0.01175747811794281
+ ],
+ "p_stock_fp16": [
+ 0.5338619947433472,
+ 0.3367025554180145,
+ 0.11767544597387314,
+ 0.01176004484295845
+ ],
+ "p_fast": [
+ 0.5350400805473328,
+ 0.335801899433136,
+ 0.11741797626018524,
+ 0.011740045621991158
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999470710754395,
+ 5.2890296501573175e-05
+ ],
+ "p_stock_fp16": [
+ 0.9999500513076782,
+ 4.990829256712459e-05
+ ],
+ "p_fast": [
+ 0.9999452829360962,
+ 5.4706317314412445e-05
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999804675579071,
+ 0.0001953149476321414
+ ],
+ "p_stock_fp16": [
+ 0.9997989535331726,
+ 0.00020110653713345528
+ ],
+ "p_fast": [
+ 0.999806821346283,
+ 0.00019316785619594157
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.2408217191696167,
+ 0.14666114747524261,
+ 0.11278093606233597,
+ 0.010207103565335274,
+ 0.007335993926972151,
+ 0.4821930229663849
+ ],
+ "p_stock_fp16": [
+ 0.24155540764331818,
+ 0.146976500749588,
+ 0.11355384439229965,
+ 0.010186702944338322,
+ 0.007337207440286875,
+ 0.4803902804851532
+ ],
+ "p_fast": [
+ 0.2399686872959137,
+ 0.14685116708278656,
+ 0.11179351061582565,
+ 0.010229084640741348,
+ 0.007353356573730707,
+ 0.48380422592163086
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.438136488199234,
+ 0.5618634819984436
+ ],
+ "p_stock_fp16": [
+ 0.441311776638031,
+ 0.558688223361969
+ ],
+ "p_fast": [
+ 0.4369823932647705,
+ 0.5630176067352295
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.006642601452767849,
+ 0.11275777220726013,
+ 0.7580914497375488,
+ 0.1225082203745842
+ ],
+ "p_stock_fp16": [
+ 0.006634419783949852,
+ 0.11232233792543411,
+ 0.758641242980957,
+ 0.12240194529294968
+ ],
+ "p_fast": [
+ 0.006652479991316795,
+ 0.11284829676151276,
+ 0.7584811449050903,
+ 0.12201811373233795
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9650728106498718,
+ 0.034927211701869965
+ ],
+ "p_stock_fp16": [
+ 0.9648054242134094,
+ 0.035194553434848785
+ ],
+ "p_fast": [
+ 0.9646725058555603,
+ 0.03532743081450462
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9728817939758301,
+ 0.02711820974946022
+ ],
+ "p_stock_fp16": [
+ 0.9731642603874207,
+ 0.026835687458515167
+ ],
+ "p_fast": [
+ 0.9728823304176331,
+ 0.02711765468120575
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.18856298923492432,
+ 0.029873721301555634,
+ 0.027661381289362907,
+ 0.04628369212150574,
+ 0.035408347845077515,
+ 0.6722098588943481
+ ],
+ "p_stock_fp16": [
+ 0.18793460726737976,
+ 0.029721055179834366,
+ 0.027649013325572014,
+ 0.04621303826570511,
+ 0.03532910719513893,
+ 0.6731531620025635
+ ],
+ "p_fast": [
+ 0.18628577888011932,
+ 0.02967686578631401,
+ 0.02744661644101143,
+ 0.04600933939218521,
+ 0.0351390540599823,
+ 0.6754423975944519
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9930188059806824,
+ 0.0069811646826565266
+ ],
+ "p_stock_fp16": [
+ 0.9930204749107361,
+ 0.006979572121053934
+ ],
+ "p_fast": [
+ 0.9929694533348083,
+ 0.007030517794191837
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5816242098808289,
+ 0.20285755395889282,
+ 0.18059970438480377,
+ 0.03491857275366783
+ ],
+ "p_stock_fp16": [
+ 0.581965446472168,
+ 0.20309557020664215,
+ 0.18010850250720978,
+ 0.0348304882645607
+ ],
+ "p_fast": [
+ 0.5821352005004883,
+ 0.20295652747154236,
+ 0.18016104400157928,
+ 0.034747205674648285
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9943746328353882,
+ 0.005625384394079447
+ ],
+ "p_stock_fp16": [
+ 0.9943581223487854,
+ 0.005641908850520849
+ ],
+ "p_fast": [
+ 0.9943348169326782,
+ 0.005665240343660116
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9930732846260071,
+ 0.006926731206476688
+ ],
+ "p_stock_fp16": [
+ 0.9932152032852173,
+ 0.006784768775105476
+ ],
+ "p_fast": [
+ 0.9934006929397583,
+ 0.0065993680618703365
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.04350755736231804,
+ 0.7518627643585205,
+ 0.012984124012291431,
+ 0.012425022199749947,
+ 0.011360721662640572,
+ 0.16785986721515656
+ ],
+ "p_stock_fp16": [
+ 0.043601907789707184,
+ 0.7520156502723694,
+ 0.012989786453545094,
+ 0.012407050468027592,
+ 0.01135204266756773,
+ 0.16763357818126678
+ ],
+ "p_fast": [
+ 0.04386109858751297,
+ 0.7505989074707031,
+ 0.013079769909381866,
+ 0.012492997571825981,
+ 0.011419525370001793,
+ 0.16854774951934814
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9840608239173889,
+ 0.015939177945256233
+ ],
+ "p_stock_fp16": [
+ 0.9840285181999207,
+ 0.015971491113305092
+ ],
+ "p_fast": [
+ 0.9840668439865112,
+ 0.015933165326714516
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.13521599769592285,
+ 0.36827537417411804,
+ 0.46942412853240967,
+ 0.02708449959754944
+ ],
+ "p_stock_fp16": [
+ 0.13542425632476807,
+ 0.36830106377601624,
+ 0.46922776103019714,
+ 0.02704690583050251
+ ],
+ "p_fast": [
+ 0.13546937704086304,
+ 0.36806416511535645,
+ 0.4693840742111206,
+ 0.027082351967692375
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.776490330696106,
+ 0.22350971400737762
+ ],
+ "p_stock_fp16": [
+ 0.7760718464851379,
+ 0.22392812371253967
+ ],
+ "p_fast": [
+ 0.7744981646537781,
+ 0.2255018800497055
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.993169903755188,
+ 0.006830038968473673
+ ],
+ "p_stock_fp16": [
+ 0.9931511878967285,
+ 0.006848819088190794
+ ],
+ "p_fast": [
+ 0.9931511878967285,
+ 0.006848819088190794
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.175027534365654,
+ 0.06410013139247894,
+ 0.20965372025966644,
+ 0.026348255574703217,
+ 0.03178912773728371,
+ 0.4930812120437622
+ ],
+ "p_stock_fp16": [
+ 0.17525389790534973,
+ 0.06423664093017578,
+ 0.20927834510803223,
+ 0.026388423517346382,
+ 0.03176840394735336,
+ 0.4930742383003235
+ ],
+ "p_fast": [
+ 0.1752185970544815,
+ 0.06419233977794647,
+ 0.2085731327533722,
+ 0.026447594165802002,
+ 0.031870752573013306,
+ 0.49369752407073975
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9969066977500916,
+ 0.0030932880472391844
+ ],
+ "p_stock_fp16": [
+ 0.9969155788421631,
+ 0.0030843650456517935
+ ],
+ "p_fast": [
+ 0.9969021081924438,
+ 0.003097907407209277
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5719269514083862,
+ 0.21320416033267975,
+ 0.1970851570367813,
+ 0.017783645540475845
+ ],
+ "p_stock_fp16": [
+ 0.5717887282371521,
+ 0.21303677558898926,
+ 0.19741199910640717,
+ 0.017762504518032074
+ ],
+ "p_fast": [
+ 0.5723857283592224,
+ 0.2126353532075882,
+ 0.19723255932331085,
+ 0.01774635724723339
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 6.416609021897557e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 6.299393451314472e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 6.293245036204098e-09
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.8054950867707475e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.82920445457313e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 1.7951385933301367e-09
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9933381080627441,
+ 0.006661858409643173
+ ],
+ "p_stock_fp16": [
+ 0.9933282136917114,
+ 0.00667178351432085
+ ],
+ "p_fast": [
+ 0.9933282136917114,
+ 0.00667178351432085
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9993873834609985,
+ 0.0006125601357780397
+ ],
+ "p_stock_fp16": [
+ 0.9993996620178223,
+ 0.0006003000307828188
+ ],
+ "p_fast": [
+ 0.9993973970413208,
+ 0.000602648186031729
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9877094030380249,
+ 0.012290582060813904
+ ],
+ "p_stock_fp16": [
+ 0.987672746181488,
+ 0.012327178381383419
+ ],
+ "p_fast": [
+ 0.9876847267150879,
+ 0.0123152956366539
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9787538051605225,
+ 0.02124619670212269
+ ],
+ "p_stock_fp16": [
+ 0.978701114654541,
+ 0.02129879966378212
+ ],
+ "p_fast": [
+ 0.9787774085998535,
+ 0.021222596988081932
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.4156995117664337,
+ 0.141651451587677,
+ 0.3883107602596283,
+ 0.05433833971619606
+ ],
+ "p_stock_fp16": [
+ 0.4152296781539917,
+ 0.14141282439231873,
+ 0.3889310956001282,
+ 0.0544264018535614
+ ],
+ "p_fast": [
+ 0.416106253862381,
+ 0.14171135425567627,
+ 0.3878536820411682,
+ 0.054328665137290955
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9987267851829529,
+ 0.0012732489267364144
+ ],
+ "p_stock_fp16": [
+ 0.9987290501594543,
+ 0.0012709128204733133
+ ],
+ "p_fast": [
+ 0.9987342953681946,
+ 0.0012656557373702526
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 6.1672627005293634e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 6.290171938871936e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 6.290171938871936e-09
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9890120029449463,
+ 0.010988010093569756
+ ],
+ "p_stock_fp16": [
+ 0.9888997077941895,
+ 0.011100264266133308
+ ],
+ "p_fast": [
+ 0.9890449047088623,
+ 0.01095515489578247
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5623424649238586,
+ 0.4376574754714966
+ ],
+ "p_stock_fp16": [
+ 0.5627773404121399,
+ 0.4372226595878601
+ ],
+ "p_fast": [
+ 0.5628975033760071,
+ 0.4371025562286377
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.6811074018478394,
+ 0.11191213130950928,
+ 0.15963312983512878,
+ 0.04734742268919945
+ ],
+ "p_stock_fp16": [
+ 0.6810320019721985,
+ 0.11213289201259613,
+ 0.1593681126832962,
+ 0.04746701568365097
+ ],
+ "p_fast": [
+ 0.6808775067329407,
+ 0.11202878504991531,
+ 0.15979944169521332,
+ 0.04729432240128517
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999995231628418,
+ 4.653498706375103e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999995231628418,
+ 4.84079805573856e-07
+ ],
+ "p_fast": [
+ 0.9999995231628418,
+ 4.6530772124242503e-07
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 4.169056655456416e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 4.256143437686433e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 4.254065988362754e-09
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9982636570930481,
+ 0.00173632160294801
+ ],
+ "p_stock_fp16": [
+ 0.9982463121414185,
+ 0.0017537595704197884
+ ],
+ "p_fast": [
+ 0.998267412185669,
+ 0.0017325180815532804
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.91415935754776,
+ 0.085840605199337
+ ],
+ "p_stock_fp16": [
+ 0.9129420518875122,
+ 0.08705801516771317
+ ],
+ "p_fast": [
+ 0.9132713079452515,
+ 0.08672870695590973
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.6391535401344299,
+ 0.22671420872211456,
+ 0.12480592727661133,
+ 0.009326225146651268
+ ],
+ "p_stock_fp16": [
+ 0.6392285227775574,
+ 0.226371169090271,
+ 0.125075101852417,
+ 0.009325223974883556
+ ],
+ "p_fast": [
+ 0.6389340758323669,
+ 0.226709246635437,
+ 0.12501749396324158,
+ 0.009339150972664356
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.928222914671096e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 2.1227489099828745e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 2.120676967365398e-08
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 8.446462973488167e-10
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 8.421865982377597e-10
+ ],
+ "p_fast": [
+ 1.0,
+ 8.41364533599176e-10
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9925096035003662,
+ 0.007490422576665878
+ ],
+ "p_stock_fp16": [
+ 0.992508590221405,
+ 0.007491440046578646
+ ],
+ "p_fast": [
+ 0.9924867153167725,
+ 0.007513254880905151
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.05382877588272095,
+ 0.9461711645126343
+ ],
+ "p_stock_fp16": [
+ 0.054048750549554825,
+ 0.9459512829780579
+ ],
+ "p_fast": [
+ 0.05384937301278114,
+ 0.9461506009101868
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.4668763279914856,
+ 0.13046501576900482,
+ 0.36059248447418213,
+ 0.04206610098481178
+ ],
+ "p_stock_fp16": [
+ 0.4662834405899048,
+ 0.13030676543712616,
+ 0.36137303709983826,
+ 0.04203672707080841
+ ],
+ "p_fast": [
+ 0.4663223326206207,
+ 0.1303812712430954,
+ 0.36105039715766907,
+ 0.0422460101544857
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.042238060384988785,
+ 0.9577619433403015
+ ],
+ "p_stock_fp16": [
+ 0.04220600053668022,
+ 0.9577940106391907
+ ],
+ "p_fast": [
+ 0.04232458770275116,
+ 0.9576753973960876
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.05428113788366318,
+ 0.9457188844680786
+ ],
+ "p_stock_fp16": [
+ 0.05414869263768196,
+ 0.9458513259887695
+ ],
+ "p_fast": [
+ 0.05429893732070923,
+ 0.9457011222839355
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7793443202972412,
+ 0.2206556499004364
+ ],
+ "p_stock_fp16": [
+ 0.7799091339111328,
+ 0.22009089589118958
+ ],
+ "p_fast": [
+ 0.7798252105712891,
+ 0.22017472982406616
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.991014301776886,
+ 0.008985652588307858
+ ],
+ "p_stock_fp16": [
+ 0.9909753203392029,
+ 0.009024644270539284
+ ],
+ "p_fast": [
+ 0.9909818768501282,
+ 0.009018097072839737
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.03797188773751259,
+ 0.11280719190835953,
+ 0.826903223991394,
+ 0.02231777459383011
+ ],
+ "p_stock_fp16": [
+ 0.037694815546274185,
+ 0.11267338693141937,
+ 0.8272820115089417,
+ 0.022349800914525986
+ ],
+ "p_fast": [
+ 0.03772996366024017,
+ 0.11291620135307312,
+ 0.8270431756973267,
+ 0.022310646250844002
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9991239905357361,
+ 0.0008760163909755647
+ ],
+ "p_stock_fp16": [
+ 0.9991164803504944,
+ 0.0008834766922518611
+ ],
+ "p_fast": [
+ 0.9991207718849182,
+ 0.0008791771251708269
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999716579914093,
+ 0.000283419038169086
+ ],
+ "p_stock_fp16": [
+ 0.9997156262397766,
+ 0.00028434465639293194
+ ],
+ "p_fast": [
+ 0.9997157454490662,
+ 0.00028420588932931423
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9937822222709656,
+ 0.006217820569872856
+ ],
+ "p_stock_fp16": [
+ 0.9937466382980347,
+ 0.006253384985029697
+ ],
+ "p_fast": [
+ 0.9937466382980347,
+ 0.006253384985029697
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 3.337593807373196e-05,
+ 0.9999666213989258
+ ],
+ "p_stock_fp16": [
+ 3.282725083408877e-05,
+ 0.9999672174453735
+ ],
+ "p_fast": [
+ 3.23183267028071e-05,
+ 0.9999676942825317
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2859868109226227,
+ 0.15410220623016357,
+ 0.5042160153388977,
+ 0.05569496750831604
+ ],
+ "p_stock_fp16": [
+ 0.28657978773117065,
+ 0.15440958738327026,
+ 0.5034545660018921,
+ 0.05555602163076401
+ ],
+ "p_fast": [
+ 0.28569144010543823,
+ 0.15419423580169678,
+ 0.5043506622314453,
+ 0.055763714015483856
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9991700649261475,
+ 0.0008299402543343604
+ ],
+ "p_stock_fp16": [
+ 0.9991657733917236,
+ 0.0008342567598447204
+ ],
+ "p_fast": [
+ 0.9991624355316162,
+ 0.0008375191246159375
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 2.6891433524411923e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 2.653042674438666e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 2.6569320077385328e-09
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9983891248703003,
+ 0.0016108963172882795
+ ],
+ "p_stock_fp16": [
+ 0.9983970522880554,
+ 0.0016029166290536523
+ ],
+ "p_fast": [
+ 0.9984103441238403,
+ 0.001589687541127205
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.0034855252597481012,
+ 0.9965144991874695
+ ],
+ "p_stock_fp16": [
+ 0.003500414779409766,
+ 0.9964995384216309
+ ],
+ "p_fast": [
+ 0.003572683082893491,
+ 0.9964273571968079
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.7407867312431335,
+ 0.08628058433532715,
+ 0.1502152681350708,
+ 0.022717468440532684
+ ],
+ "p_stock_fp16": [
+ 0.7403666377067566,
+ 0.08583977073431015,
+ 0.15115083754062653,
+ 0.022642774507403374
+ ],
+ "p_fast": [
+ 0.7421941161155701,
+ 0.08581039309501648,
+ 0.14953942596912384,
+ 0.022456135600805283
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 6.434143218214672e-10
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 6.435235677670903e-10
+ ],
+ "p_fast": [
+ 1.0,
+ 6.539768171442972e-10
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 6.620669568135895e-10
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 6.681791231422096e-10
+ ],
+ "p_fast": [
+ 1.0,
+ 6.527007823109443e-10
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 8.731669276507148e-10
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 8.723212707728578e-10
+ ],
+ "p_fast": [
+ 1.0,
+ 8.714697852241216e-10
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9936228394508362,
+ 0.006377215962857008
+ ],
+ "p_stock_fp16": [
+ 0.9936925172805786,
+ 0.006307474337518215
+ ],
+ "p_fast": [
+ 0.9936332702636719,
+ 0.006366661284118891
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.7317870259284973,
+ 0.16596166789531708,
+ 0.09646870195865631,
+ 0.00578261585906148
+ ],
+ "p_stock_fp16": [
+ 0.7281661033630371,
+ 0.16779713332653046,
+ 0.09821031242609024,
+ 0.0058264280669391155
+ ],
+ "p_fast": [
+ 0.7321892976760864,
+ 0.16497677564620972,
+ 0.09703220427036285,
+ 0.005801686085760593
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9835695624351501,
+ 0.016430504620075226
+ ],
+ "p_stock_fp16": [
+ 0.983472466468811,
+ 0.0165275726467371
+ ],
+ "p_fast": [
+ 0.9835357666015625,
+ 0.016464198008179665
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.4016177551923192e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.39281270961078e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 1.3996302783425563e-08
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999994158744812,
+ 5.860231794940773e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999940395355225,
+ 5.920348030485911e-06
+ ],
+ "p_fast": [
+ 0.9999940395355225,
+ 5.917457656323677e-06
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.942081868648529,
+ 0.05791809782385826
+ ],
+ "p_stock_fp16": [
+ 0.941439151763916,
+ 0.058560825884342194
+ ],
+ "p_fast": [
+ 0.9421883225440979,
+ 0.05781161040067673
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.4360455274581909,
+ 0.16167399287223816,
+ 0.3795865476131439,
+ 0.02269396185874939
+ ],
+ "p_stock_fp16": [
+ 0.43586602807044983,
+ 0.16168269515037537,
+ 0.3797980546951294,
+ 0.022653257474303246
+ ],
+ "p_fast": [
+ 0.4355769753456116,
+ 0.1614965945482254,
+ 0.3802882134914398,
+ 0.02263823337852955
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.2801613991086924e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.2843729635392265e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 1.2613772248926125e-08
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 8.76037853370093e-10
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 8.70619187853805e-10
+ ],
+ "p_fast": [
+ 1.0,
+ 8.843294985183547e-10
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9985772371292114,
+ 0.0014227192150428891
+ ],
+ "p_stock_fp16": [
+ 0.9985792636871338,
+ 0.00142071139998734
+ ],
+ "p_fast": [
+ 0.9985910058021545,
+ 0.0014089837204664946
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.03020140342414379,
+ 0.9697986245155334
+ ],
+ "p_stock_fp16": [
+ 0.03035798855125904,
+ 0.9696420431137085
+ ],
+ "p_fast": [
+ 0.03058880753815174,
+ 0.9694111943244934
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.812294602394104,
+ 0.14415371417999268,
+ 0.03713776916265488,
+ 0.006413840223103762
+ ],
+ "p_stock_fp16": [
+ 0.8119589686393738,
+ 0.1444433331489563,
+ 0.037214040756225586,
+ 0.006383692380040884
+ ],
+ "p_fast": [
+ 0.8108994960784912,
+ 0.14524434506893158,
+ 0.03739301487803459,
+ 0.006463124882429838
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.998994767665863,
+ 0.0010052593424916267
+ ],
+ "p_stock_fp16": [
+ 0.9989926218986511,
+ 0.0010073602898046374
+ ],
+ "p_fast": [
+ 0.9989960789680481,
+ 0.0010039265034720302
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 2.0498280406400227e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 2.0609581596886528e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 2.0290059410399408e-08
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999674558639526,
+ 3.2552608900005e-05
+ ],
+ "p_stock_fp16": [
+ 0.9999704360961914,
+ 2.9613634978886694e-05
+ ],
+ "p_fast": [
+ 0.999969482421875,
+ 3.053872933378443e-05
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9990270137786865,
+ 0.0009729345911182463
+ ],
+ "p_stock_fp16": [
+ 0.9990247488021851,
+ 0.0009752073092386127
+ ],
+ "p_fast": [
+ 0.9990242719650269,
+ 0.0009756832732819021
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5065508484840393,
+ 0.34830689430236816,
+ 0.1002361997961998,
+ 0.044906094670295715
+ ],
+ "p_stock_fp16": [
+ 0.5065916776657104,
+ 0.34851521253585815,
+ 0.10019315779209137,
+ 0.04469988867640495
+ ],
+ "p_fast": [
+ 0.50690096616745,
+ 0.34804755449295044,
+ 0.10019314289093018,
+ 0.04485839605331421
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999986886978149,
+ 1.319354964834929e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999986886978149,
+ 1.35432344450237e-06
+ ],
+ "p_fast": [
+ 0.9999986886978149,
+ 1.3333269635040779e-06
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.2827026107942174e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.284808370805024e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 1.2854357578362396e-09
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 8.607913826175206e-10
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 8.723212707728578e-10
+ ],
+ "p_fast": [
+ 1.0,
+ 8.592166422793923e-10
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.1084574369490952e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.1211771511199231e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 1.1222726081783208e-09
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.691309928894043,
+ 0.2594248950481415,
+ 0.04071633145213127,
+ 0.008548864163458347
+ ],
+ "p_stock_fp16": [
+ 0.6919106841087341,
+ 0.258800745010376,
+ 0.040738821029663086,
+ 0.008549750782549381
+ ],
+ "p_fast": [
+ 0.691299557685852,
+ 0.2593308091163635,
+ 0.04080233350396156,
+ 0.008567261509597301
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 9.896183872371012e-10
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 9.838949655005536e-10
+ ],
+ "p_fast": [
+ 1.0,
+ 9.89676895990499e-10
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5140857100486755,
+ 0.48591428995132446
+ ],
+ "p_stock_fp16": [
+ 0.5172051191329956,
+ 0.4827949106693268
+ ],
+ "p_fast": [
+ 0.5103134512901306,
+ 0.489686518907547
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9982128143310547,
+ 0.0017872567987069488
+ ],
+ "p_stock_fp16": [
+ 0.9981858134269714,
+ 0.0018141856417059898
+ ],
+ "p_fast": [
+ 0.9982012510299683,
+ 0.0017987774917855859
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.21405240893363953,
+ 0.3071655035018921,
+ 0.44633322954177856,
+ 0.03244880959391594
+ ],
+ "p_stock_fp16": [
+ 0.21352431178092957,
+ 0.30765682458877563,
+ 0.4463285803794861,
+ 0.032490190118551254
+ ],
+ "p_fast": [
+ 0.213642418384552,
+ 0.30722635984420776,
+ 0.44657543301582336,
+ 0.03255581855773926
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.8440533876419067,
+ 0.0181967094540596,
+ 0.014200989156961441,
+ 0.04481679946184158,
+ 0.012798666954040527,
+ 0.0659334659576416
+ ],
+ "p_stock_fp16": [
+ 0.8441758751869202,
+ 0.01805882342159748,
+ 0.014188380911946297,
+ 0.04493672773241997,
+ 0.012793122790753841,
+ 0.06584711372852325
+ ],
+ "p_fast": [
+ 0.844370424747467,
+ 0.01820465363562107,
+ 0.014163958840072155,
+ 0.044859375804662704,
+ 0.0127960704267025,
+ 0.06560550630092621
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.430278673275609e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.4264998071666923e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 1.4511766233127332e-09
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.994398295879364,
+ 0.0056016952730715275
+ ],
+ "p_stock_fp16": [
+ 0.9944180250167847,
+ 0.005581962410360575
+ ],
+ "p_fast": [
+ 0.9943745136260986,
+ 0.005625496152788401
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9671284556388855,
+ 0.03287157043814659
+ ],
+ "p_stock_fp16": [
+ 0.9671087265014648,
+ 0.032891273498535156
+ ],
+ "p_fast": [
+ 0.9671630263328552,
+ 0.03283695876598358
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.07449834793806076,
+ 0.06760001182556152,
+ 0.46635618805885315,
+ 0.39154553413391113
+ ],
+ "p_stock_fp16": [
+ 0.07454366236925125,
+ 0.06763293594121933,
+ 0.4660966098308563,
+ 0.3917268216609955
+ ],
+ "p_fast": [
+ 0.07399621605873108,
+ 0.06738255172967911,
+ 0.46653011441230774,
+ 0.39209115505218506
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.354805588722229,
+ 0.18484385311603546,
+ 0.002548832446336746,
+ 0.0034615856129676104,
+ 0.447773277759552,
+ 0.006566840223968029
+ ],
+ "p_stock_fp16": [
+ 0.35511288046836853,
+ 0.18513189256191254,
+ 0.002552029211074114,
+ 0.0034678312949836254,
+ 0.44715461134910583,
+ 0.00658078771084547
+ ],
+ "p_fast": [
+ 0.35582149028778076,
+ 0.18441759049892426,
+ 0.0025521323550492525,
+ 0.003467971459031105,
+ 0.4471726715564728,
+ 0.006568212527781725
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 5.830121718730652e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 5.850841588994626e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 5.8520024381891744e-09
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9530665874481201,
+ 0.0469333790242672
+ ],
+ "p_stock_fp16": [
+ 0.9529107809066772,
+ 0.047089237719774246
+ ],
+ "p_fast": [
+ 0.9527283310890198,
+ 0.04727169871330261
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.963582456111908,
+ 0.03641754761338234
+ ],
+ "p_stock_fp16": [
+ 0.9634501338005066,
+ 0.036549877375364304
+ ],
+ "p_fast": [
+ 0.9634659290313721,
+ 0.03653402253985405
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.4077551066875458,
+ 0.49678829312324524,
+ 0.0857841894030571,
+ 0.009672419168055058
+ ],
+ "p_stock_fp16": [
+ 0.4081886112689972,
+ 0.4962311387062073,
+ 0.08589585870504379,
+ 0.009684419259428978
+ ],
+ "p_fast": [
+ 0.4081200957298279,
+ 0.49614787101745605,
+ 0.08604934066534042,
+ 0.009682794101536274
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.7600809931755066,
+ 0.012381480075418949,
+ 0.03508656099438667,
+ 0.06916041672229767,
+ 0.032925304025411606,
+ 0.09036515653133392
+ ],
+ "p_stock_fp16": [
+ 0.760770857334137,
+ 0.012393159791827202,
+ 0.03503008559346199,
+ 0.06885400414466858,
+ 0.03290771692991257,
+ 0.0900440514087677
+ ],
+ "p_fast": [
+ 0.7603712677955627,
+ 0.012422991916537285,
+ 0.03511440381407738,
+ 0.06901974976062775,
+ 0.03298693150281906,
+ 0.09008468687534332
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.039916276931762695,
+ 0.9600837826728821
+ ],
+ "p_stock_fp16": [
+ 0.04008735716342926,
+ 0.9599125981330872
+ ],
+ "p_fast": [
+ 0.039787814021110535,
+ 0.960212230682373
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.005634257569909096,
+ 0.9943658113479614
+ ],
+ "p_stock_fp16": [
+ 0.005674875341355801,
+ 0.9943251013755798
+ ],
+ "p_fast": [
+ 0.0056145815178751945,
+ 0.9943854808807373
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9976844787597656,
+ 0.00231552729383111
+ ],
+ "p_stock_fp16": [
+ 0.9976818561553955,
+ 0.0023180751595646143
+ ],
+ "p_fast": [
+ 0.9976813793182373,
+ 0.0023186402395367622
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.17886225879192352,
+ 0.11180384457111359,
+ 0.3105556070804596,
+ 0.3987782299518585
+ ],
+ "p_stock_fp16": [
+ 0.17875096201896667,
+ 0.11158672720193863,
+ 0.3103657066822052,
+ 0.3992965817451477
+ ],
+ "p_fast": [
+ 0.1791982501745224,
+ 0.11192058026790619,
+ 0.3105352520942688,
+ 0.398345947265625
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.15747618675231934,
+ 0.16297447681427002,
+ 0.03193182125687599,
+ 0.17153778672218323,
+ 0.4629267156124115,
+ 0.013153048232197762
+ ],
+ "p_stock_fp16": [
+ 0.15681461989879608,
+ 0.1627432405948639,
+ 0.031897738575935364,
+ 0.17088687419891357,
+ 0.4645187258720398,
+ 0.013138832524418831
+ ],
+ "p_fast": [
+ 0.15708954632282257,
+ 0.16271044313907623,
+ 0.031945858150720596,
+ 0.17252910137176514,
+ 0.4626144766807556,
+ 0.013110553845763206
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9923869371414185,
+ 0.007613067980855703
+ ],
+ "p_stock_fp16": [
+ 0.9923350214958191,
+ 0.007664946373552084
+ ],
+ "p_fast": [
+ 0.9922749400138855,
+ 0.007725066039711237
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9881777167320251,
+ 0.011822234839200974
+ ],
+ "p_stock_fp16": [
+ 0.9882221817970276,
+ 0.011777842417359352
+ ],
+ "p_fast": [
+ 0.9882732033729553,
+ 0.0117268031463027
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9989345669746399,
+ 0.0010654883226379752
+ ],
+ "p_stock_fp16": [
+ 0.9989271759986877,
+ 0.001072782906703651
+ ],
+ "p_fast": [
+ 0.9989271759986877,
+ 0.001072782906703651
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.031215203925967216,
+ 0.14637240767478943,
+ 0.6154507398605347,
+ 0.2069617211818695
+ ],
+ "p_stock_fp16": [
+ 0.03112652339041233,
+ 0.14619413018226624,
+ 0.6151997447013855,
+ 0.2074795961380005
+ ],
+ "p_fast": [
+ 0.031195776537060738,
+ 0.1463049054145813,
+ 0.6150650978088379,
+ 0.20743417739868164
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.1643088012933731,
+ 0.11127668619155884,
+ 0.1488676518201828,
+ 0.12384389340877533,
+ 0.07701191306114197,
+ 0.3746910095214844
+ ],
+ "p_stock_fp16": [
+ 0.16421177983283997,
+ 0.11078621447086334,
+ 0.14937058091163635,
+ 0.12359094619750977,
+ 0.07688938826322556,
+ 0.3751510977745056
+ ],
+ "p_fast": [
+ 0.16410678625106812,
+ 0.11136600375175476,
+ 0.14942093193531036,
+ 0.12375339865684509,
+ 0.07699046283960342,
+ 0.37436243891716003
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.5330689495662853e-10,
+ 1.0
+ ],
+ "p_stock_fp16": [
+ 1.5344847614784385e-10,
+ 1.0
+ ],
+ "p_fast": [
+ 1.5106947637288926e-10,
+ 1.0
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.027207739651203156,
+ 0.9727923274040222
+ ],
+ "p_stock_fp16": [
+ 0.027324529364705086,
+ 0.9726754426956177
+ ],
+ "p_fast": [
+ 0.027195055037736893,
+ 0.9728049635887146
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9184237718582153,
+ 0.08157625794410706
+ ],
+ "p_stock_fp16": [
+ 0.9185898900032043,
+ 0.08141013979911804
+ ],
+ "p_fast": [
+ 0.91880863904953,
+ 0.08119132369756699
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2651115357875824,
+ 0.2292918711900711,
+ 0.3461720049381256,
+ 0.1594245880842209
+ ],
+ "p_stock_fp16": [
+ 0.2648496925830841,
+ 0.2296564280986786,
+ 0.3461044728755951,
+ 0.15938936173915863
+ ],
+ "p_fast": [
+ 0.2641312777996063,
+ 0.2288099229335785,
+ 0.34685513377189636,
+ 0.1602037101984024
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.5090178847312927,
+ 0.04319735988974571,
+ 0.012556241825222969,
+ 0.08478369563817978,
+ 0.28864970803260803,
+ 0.06179504469037056
+ ],
+ "p_stock_fp16": [
+ 0.5078005194664001,
+ 0.04325868934392929,
+ 0.01258899550884962,
+ 0.08511096984148026,
+ 0.28933605551719666,
+ 0.061904728412628174
+ ],
+ "p_fast": [
+ 0.5086544156074524,
+ 0.043162502348423004,
+ 0.012573275715112686,
+ 0.08515007793903351,
+ 0.2886926531791687,
+ 0.061767078936100006
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.0006522627081722021,
+ 0.9993477463722229
+ ],
+ "p_stock_fp16": [
+ 0.0006538155139423907,
+ 0.9993461966514587
+ ],
+ "p_fast": [
+ 0.0006449432112276554,
+ 0.9993550181388855
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.013501730747520924,
+ 0.986498236656189
+ ],
+ "p_stock_fp16": [
+ 0.014531521126627922,
+ 0.9854684472084045
+ ],
+ "p_fast": [
+ 0.014104306697845459,
+ 0.9858956933021545
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.03638250008225441,
+ 0.9636174440383911
+ ],
+ "p_stock_fp16": [
+ 0.03711691126227379,
+ 0.9628831148147583
+ ],
+ "p_fast": [
+ 0.036804068833589554,
+ 0.9631959199905396
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.13897722959518433,
+ 0.1299632042646408,
+ 0.26211851835250854,
+ 0.4689411222934723
+ ],
+ "p_stock_fp16": [
+ 0.13877592980861664,
+ 0.12998653948307037,
+ 0.2624518871307373,
+ 0.46878570318222046
+ ],
+ "p_fast": [
+ 0.13900431990623474,
+ 0.12988299131393433,
+ 0.26224276423454285,
+ 0.4688699245452881
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.143416628241539,
+ 0.18449728190898895,
+ 0.004301834385842085,
+ 0.004663313273340464,
+ 0.6565998792648315,
+ 0.006521086674183607
+ ],
+ "p_stock_fp16": [
+ 0.14572446048259735,
+ 0.18592993915081024,
+ 0.004330159164965153,
+ 0.004691172391176224,
+ 0.6527729034423828,
+ 0.006551322061568499
+ ],
+ "p_fast": [
+ 0.14560580253601074,
+ 0.18650567531585693,
+ 0.004360567778348923,
+ 0.004714897833764553,
+ 0.6522414088249207,
+ 0.006571608129888773
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 2.7456767415223737e-10,
+ 1.0
+ ],
+ "p_stock_fp16": [
+ 2.9577967852745246e-10,
+ 1.0
+ ],
+ "p_fast": [
+ 2.395297016288822e-10,
+ 1.0
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.0008434649789705873,
+ 0.9991565942764282
+ ],
+ "p_stock_fp16": [
+ 0.0008377236081287265,
+ 0.9991623163223267
+ ],
+ "p_fast": [
+ 0.0008426422136835754,
+ 0.9991573095321655
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9911888837814331,
+ 0.008811095729470253
+ ],
+ "p_stock_fp16": [
+ 0.9912527203559875,
+ 0.008747299201786518
+ ],
+ "p_fast": [
+ 0.9910968542098999,
+ 0.008903193287551403
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.17365014553070068,
+ 0.5990110039710999,
+ 0.1895761787891388,
+ 0.03776267170906067
+ ],
+ "p_stock_fp16": [
+ 0.17374032735824585,
+ 0.5981789231300354,
+ 0.1902582049369812,
+ 0.037822477519512177
+ ],
+ "p_fast": [
+ 0.17253172397613525,
+ 0.599847137928009,
+ 0.18985947966575623,
+ 0.03776165097951889
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.4388510584831238,
+ 0.17121917009353638,
+ 0.1909327656030655,
+ 0.05251241847872734,
+ 0.05318012461066246,
+ 0.09330441057682037
+ ],
+ "p_stock_fp16": [
+ 0.43873026967048645,
+ 0.17082634568214417,
+ 0.19078004360198975,
+ 0.05260394141077995,
+ 0.053693875670433044,
+ 0.09336546063423157
+ ],
+ "p_fast": [
+ 0.437713623046875,
+ 0.17168334126472473,
+ 0.1914798617362976,
+ 0.05258464813232422,
+ 0.05284203961491585,
+ 0.09369651228189468
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 7.898071086032132e-10
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 7.967824733334794e-10
+ ],
+ "p_fast": [
+ 1.0,
+ 7.86922971229842e-10
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9991130232810974,
+ 0.0008869627490639687
+ ],
+ "p_stock_fp16": [
+ 0.9991034865379333,
+ 0.0008965019369497895
+ ],
+ "p_fast": [
+ 0.9991078972816467,
+ 0.0008921389817260206
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8698384165763855,
+ 0.1301615834236145
+ ],
+ "p_stock_fp16": [
+ 0.8712559342384338,
+ 0.12874406576156616
+ ],
+ "p_fast": [
+ 0.8713106513023376,
+ 0.12868930399417877
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.048659924417734146,
+ 0.33810868859291077,
+ 0.285380095243454,
+ 0.32785123586654663
+ ],
+ "p_stock_fp16": [
+ 0.04891016334295273,
+ 0.33908963203430176,
+ 0.2852633595466614,
+ 0.32673683762550354
+ ],
+ "p_fast": [
+ 0.048707813024520874,
+ 0.3411671817302704,
+ 0.2839445173740387,
+ 0.32618042826652527
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.6906778812408447,
+ 0.03893024101853371,
+ 0.01991366595029831,
+ 0.02889903634786606,
+ 0.15194302797317505,
+ 0.06963618099689484
+ ],
+ "p_stock_fp16": [
+ 0.6881656050682068,
+ 0.03928128629922867,
+ 0.019965194165706635,
+ 0.028992503881454468,
+ 0.15411385893821716,
+ 0.0694815143942833
+ ],
+ "p_fast": [
+ 0.6889662742614746,
+ 0.0392119437456131,
+ 0.019891060888767242,
+ 0.028969598934054375,
+ 0.15346667170524597,
+ 0.06949445605278015
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.002468275371938944,
+ 0.9975316524505615
+ ],
+ "p_stock_fp16": [
+ 0.0025200408417731524,
+ 0.9974799752235413
+ ],
+ "p_fast": [
+ 0.002452233573421836,
+ 0.9975478053092957
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9641194939613342,
+ 0.035880494862794876
+ ],
+ "p_stock_fp16": [
+ 0.9642037153244019,
+ 0.035796333104372025
+ ],
+ "p_fast": [
+ 0.9642037153244019,
+ 0.035796333104372025
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9896026253700256,
+ 0.010397412814199924
+ ],
+ "p_stock_fp16": [
+ 0.9895979166030884,
+ 0.010402029380202293
+ ],
+ "p_fast": [
+ 0.989591658115387,
+ 0.010408313944935799
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.46187058091163635,
+ 0.38308459520339966,
+ 0.09633477032184601,
+ 0.058710064738988876
+ ],
+ "p_stock_fp16": [
+ 0.4613054394721985,
+ 0.3831833004951477,
+ 0.09664767235517502,
+ 0.05886358022689819
+ ],
+ "p_fast": [
+ 0.46156439185142517,
+ 0.3833984136581421,
+ 0.09651323407888412,
+ 0.05852394551038742
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.07601017504930496,
+ 0.19812172651290894,
+ 0.10501913726329803,
+ 0.02682625874876976,
+ 0.2704060971736908,
+ 0.323616623878479
+ ],
+ "p_stock_fp16": [
+ 0.07597943395376205,
+ 0.19799195230007172,
+ 0.10497318208217621,
+ 0.026854239404201508,
+ 0.27046602964401245,
+ 0.3237350881099701
+ ],
+ "p_fast": [
+ 0.0761413648724556,
+ 0.19793011248111725,
+ 0.10545404255390167,
+ 0.026754247024655342,
+ 0.26973047852516174,
+ 0.32398971915245056
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.4141549042889778e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.4026707573222552e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 1.4026707573222552e-09
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9773577451705933,
+ 0.022642232477664948
+ ],
+ "p_stock_fp16": [
+ 0.9775683879852295,
+ 0.02243161015212536
+ ],
+ "p_fast": [
+ 0.9774032235145569,
+ 0.022596824914216995
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9663182497024536,
+ 0.033681679517030716
+ ],
+ "p_stock_fp16": [
+ 0.9663628935813904,
+ 0.033637091517448425
+ ],
+ "p_fast": [
+ 0.9664223790168762,
+ 0.03357761725783348
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.09275461733341217,
+ 0.17282764613628387,
+ 0.26986631751060486,
+ 0.4645513892173767
+ ],
+ "p_stock_fp16": [
+ 0.0928221344947815,
+ 0.17273849248886108,
+ 0.2699044346809387,
+ 0.4645349383354187
+ ],
+ "p_fast": [
+ 0.09271449595689774,
+ 0.1726224422454834,
+ 0.26998665928840637,
+ 0.4646764099597931
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.3107568025588989,
+ 0.12133712321519852,
+ 0.01833208091557026,
+ 0.010915376245975494,
+ 0.5036391615867615,
+ 0.03501947224140167
+ ],
+ "p_stock_fp16": [
+ 0.31031283736228943,
+ 0.12122394144535065,
+ 0.01832890510559082,
+ 0.010934006422758102,
+ 0.5041795372962952,
+ 0.03502073511481285
+ ],
+ "p_fast": [
+ 0.31126871705055237,
+ 0.12153798341751099,
+ 0.01838536560535431,
+ 0.010935602709650993,
+ 0.5027779936790466,
+ 0.03509432077407837
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.293394391588265e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.2744987287760523e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 1.2882633848576575e-09
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999998807907104,
+ 1.2043842900766322e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999998807907104,
+ 1.171037951053222e-07
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 1.1661876442303765e-07
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.6907447886183036e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.674320238009841e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 1.6204314334800074e-08
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5077562928199768,
+ 0.4375225305557251,
+ 0.03748030588030815,
+ 0.017240874469280243
+ ],
+ "p_stock_fp16": [
+ 0.507655918598175,
+ 0.43762674927711487,
+ 0.03748147934675217,
+ 0.017235849052667618
+ ],
+ "p_fast": [
+ 0.5070186257362366,
+ 0.4379318654537201,
+ 0.037751056253910065,
+ 0.017298467457294464
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.27183154225349426,
+ 0.24665230512619019,
+ 0.09246212989091873,
+ 0.05073807016015053,
+ 0.1566755771636963,
+ 0.18164046108722687
+ ],
+ "p_stock_fp16": [
+ 0.272569864988327,
+ 0.245987206697464,
+ 0.09237990528345108,
+ 0.05071897804737091,
+ 0.15645445883274078,
+ 0.18188954889774323
+ ],
+ "p_fast": [
+ 0.2718876898288727,
+ 0.24651993811130524,
+ 0.09237676858901978,
+ 0.05076680704951286,
+ 0.15683157742023468,
+ 0.1816171407699585
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.06327264755964279,
+ 0.7663608193397522,
+ 0.1492958515882492,
+ 0.021070657297968864
+ ],
+ "p_stock_fp16": [
+ 0.0629885196685791,
+ 0.7666081786155701,
+ 0.14930474758148193,
+ 0.021098511293530464
+ ],
+ "p_fast": [
+ 0.06324407458305359,
+ 0.766343355178833,
+ 0.14936251938343048,
+ 0.021050067618489265
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.08604495972394943,
+ 0.24704822897911072,
+ 0.2550152540206909,
+ 0.09231465309858322,
+ 0.05258480831980705,
+ 0.26699212193489075
+ ],
+ "p_stock_fp16": [
+ 0.08626484870910645,
+ 0.2457456886768341,
+ 0.2555350661277771,
+ 0.0925486758351326,
+ 0.0526297502219677,
+ 0.26727595925331116
+ ],
+ "p_fast": [
+ 0.08597361296415329,
+ 0.2473195195198059,
+ 0.25492116808891296,
+ 0.09259723126888275,
+ 0.05255461484193802,
+ 0.2666338384151459
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.99910968542099,
+ 0.0008903192356228828
+ ],
+ "p_stock_fp16": [
+ 0.9991045594215393,
+ 0.0008954091463238001
+ ],
+ "p_fast": [
+ 0.9991087317466736,
+ 0.0008912690100260079
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7883598208427429,
+ 0.21164019405841827
+ ],
+ "p_stock_fp16": [
+ 0.7884203195571899,
+ 0.21157969534397125
+ ],
+ "p_fast": [
+ 0.7876046895980835,
+ 0.21239535510540009
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.025481287389993668,
+ 0.6571927666664124,
+ 0.29016897082328796,
+ 0.027156967669725418
+ ],
+ "p_stock_fp16": [
+ 0.025495193898677826,
+ 0.6562467217445374,
+ 0.2910655438899994,
+ 0.027192551642656326
+ ],
+ "p_fast": [
+ 0.02555827982723713,
+ 0.6565868854522705,
+ 0.2906481921672821,
+ 0.027206648141145706
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9360402226448059,
+ 0.00850815698504448,
+ 0.01660247892141342,
+ 0.010453121736645699,
+ 0.011118200607597828,
+ 0.017277900129556656
+ ],
+ "p_stock_fp16": [
+ 0.9357619285583496,
+ 0.008551003411412239,
+ 0.016709383577108383,
+ 0.010517912916839123,
+ 0.011152610182762146,
+ 0.01730727218091488
+ ],
+ "p_fast": [
+ 0.936115026473999,
+ 0.008487659506499767,
+ 0.0166180320084095,
+ 0.010419628582894802,
+ 0.011113321408629417,
+ 0.017246302217245102
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9729970097541809,
+ 0.027002960443496704
+ ],
+ "p_stock_fp16": [
+ 0.9730812907218933,
+ 0.026918698102235794
+ ],
+ "p_fast": [
+ 0.9730556607246399,
+ 0.02694428712129593
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7918785810470581,
+ 0.20812146365642548
+ ],
+ "p_stock_fp16": [
+ 0.7944647073745728,
+ 0.20553529262542725
+ ],
+ "p_fast": [
+ 0.7943052053451538,
+ 0.2056947946548462
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.056967273354530334,
+ 0.6567702889442444,
+ 0.2646607458591461,
+ 0.021601738408207893
+ ],
+ "p_stock_fp16": [
+ 0.0572441928088665,
+ 0.6572076082229614,
+ 0.2639845609664917,
+ 0.02156362310051918
+ ],
+ "p_fast": [
+ 0.05697541683912277,
+ 0.6566018462181091,
+ 0.26477348804473877,
+ 0.021649198606610298
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.10250363498926163,
+ 0.12156710773706436,
+ 0.1577567756175995,
+ 0.13974647223949432,
+ 0.08337423205375671,
+ 0.3950518071651459
+ ],
+ "p_stock_fp16": [
+ 0.10239048302173615,
+ 0.12135446071624756,
+ 0.15781304240226746,
+ 0.13967813551425934,
+ 0.08356868475675583,
+ 0.3951951861381531
+ ],
+ "p_fast": [
+ 0.1025984138250351,
+ 0.12136363983154297,
+ 0.15782496333122253,
+ 0.13996176421642303,
+ 0.08341191709041595,
+ 0.39483925700187683
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9827491044998169,
+ 0.017250876873731613
+ ],
+ "p_stock_fp16": [
+ 0.9826960563659668,
+ 0.01730390451848507
+ ],
+ "p_fast": [
+ 0.9826856851577759,
+ 0.017314286902546883
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9361225366592407,
+ 0.06387747079133987
+ ],
+ "p_stock_fp16": [
+ 0.9362266659736633,
+ 0.0637732744216919
+ ],
+ "p_fast": [
+ 0.9360808730125427,
+ 0.06391920149326324
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.18864065408706665,
+ 0.6919176578521729,
+ 0.11256653815507889,
+ 0.006875165738165379
+ ],
+ "p_stock_fp16": [
+ 0.1883215755224228,
+ 0.6922234296798706,
+ 0.11256182193756104,
+ 0.006893188692629337
+ ],
+ "p_fast": [
+ 0.18846960365772247,
+ 0.6920914053916931,
+ 0.1125403493642807,
+ 0.006898608058691025
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.24721229076385498,
+ 0.2000112682580948,
+ 0.23914487659931183,
+ 0.09753073006868362,
+ 0.07444898784160614,
+ 0.14165180921554565
+ ],
+ "p_stock_fp16": [
+ 0.24638673663139343,
+ 0.20011547207832336,
+ 0.23974089324474335,
+ 0.09743330627679825,
+ 0.07455893605947495,
+ 0.14176462590694427
+ ],
+ "p_fast": [
+ 0.24648965895175934,
+ 0.19980843365192413,
+ 0.24007534980773926,
+ 0.09756922721862793,
+ 0.07437187433242798,
+ 0.1416853964328766
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9937706589698792,
+ 0.00622929260134697
+ ],
+ "p_stock_fp16": [
+ 0.9937791228294373,
+ 0.006220849696546793
+ ],
+ "p_fast": [
+ 0.9937738180160522,
+ 0.006226134020835161
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9986302852630615,
+ 0.0013697878457605839
+ ],
+ "p_stock_fp16": [
+ 0.9986273050308228,
+ 0.0013727032346650958
+ ],
+ "p_fast": [
+ 0.998625636100769,
+ 0.0013743777526542544
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.38456177711486816,
+ 0.5569770932197571,
+ 0.04075850173830986,
+ 0.017702685669064522
+ ],
+ "p_stock_fp16": [
+ 0.384881854057312,
+ 0.5567280650138855,
+ 0.0406852662563324,
+ 0.017704781144857407
+ ],
+ "p_fast": [
+ 0.38442620635032654,
+ 0.5571561455726624,
+ 0.04071654751896858,
+ 0.017701098695397377
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.032346028834581375,
+ 0.16453826427459717,
+ 0.1629648506641388,
+ 0.14597390592098236,
+ 0.028840484097599983,
+ 0.4653364419937134
+ ],
+ "p_stock_fp16": [
+ 0.032184164971113205,
+ 0.16376420855522156,
+ 0.16376420855522156,
+ 0.1459396779537201,
+ 0.028737224638462067,
+ 0.465610533952713
+ ],
+ "p_fast": [
+ 0.0322839729487896,
+ 0.16395153105258942,
+ 0.16331234574317932,
+ 0.14639224112033844,
+ 0.028826342895627022,
+ 0.46523356437683105
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9932069182395935,
+ 0.006793042179197073
+ ],
+ "p_stock_fp16": [
+ 0.9931528568267822,
+ 0.006847159005701542
+ ],
+ "p_fast": [
+ 0.9931504130363464,
+ 0.006849650293588638
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999996542930603,
+ 3.430791821301682e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999966621398926,
+ 3.3716746656864416e-06
+ ],
+ "p_fast": [
+ 0.999996542930603,
+ 3.4727625006780727e-06
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.20391005277633667,
+ 0.645556628704071,
+ 0.13207192718982697,
+ 0.01846134103834629
+ ],
+ "p_stock_fp16": [
+ 0.20406149327754974,
+ 0.6456624269485474,
+ 0.1318163424730301,
+ 0.018459688872098923
+ ],
+ "p_fast": [
+ 0.20333434641361237,
+ 0.6458798050880432,
+ 0.13224759697914124,
+ 0.018538175150752068
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.01800079643726349,
+ 0.04048068821430206,
+ 0.02068852074444294,
+ 0.04988718032836914,
+ 0.02618858776986599,
+ 0.8447542190551758
+ ],
+ "p_stock_fp16": [
+ 0.018053831532597542,
+ 0.040526315569877625,
+ 0.020820453763008118,
+ 0.050337404012680054,
+ 0.02631952241063118,
+ 0.8439425230026245
+ ],
+ "p_fast": [
+ 0.018030550330877304,
+ 0.04031626135110855,
+ 0.020793607458472252,
+ 0.05017440393567085,
+ 0.026183104142546654,
+ 0.8445020914077759
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.11338722705841064,
+ 0.8866127729415894
+ ],
+ "p_stock_fp16": [
+ 0.11407211422920227,
+ 0.8859278559684753
+ ],
+ "p_fast": [
+ 0.1139734610915184,
+ 0.8860265612602234
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.015806756913661957,
+ 0.9841932654380798
+ ],
+ "p_stock_fp16": [
+ 0.015739111229777336,
+ 0.9842609167098999
+ ],
+ "p_fast": [
+ 0.01572398841381073,
+ 0.9842759370803833
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.15774409472942352,
+ 0.5577811002731323,
+ 0.1785818189382553,
+ 0.10589306801557541
+ ],
+ "p_stock_fp16": [
+ 0.1571502685546875,
+ 0.557690441608429,
+ 0.17903362214565277,
+ 0.10612571239471436
+ ],
+ "p_fast": [
+ 0.1578477919101715,
+ 0.5579819083213806,
+ 0.17842884361743927,
+ 0.10574138909578323
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.5800089240074158,
+ 0.1417265236377716,
+ 0.14176353812217712,
+ 0.04529455304145813,
+ 0.03233008831739426,
+ 0.058876413851976395
+ ],
+ "p_stock_fp16": [
+ 0.5790936350822449,
+ 0.1416015326976776,
+ 0.14243365824222565,
+ 0.04539135470986366,
+ 0.032566703855991364,
+ 0.05891311913728714
+ ],
+ "p_fast": [
+ 0.5770910978317261,
+ 0.14267069101333618,
+ 0.14322911202907562,
+ 0.04555578529834747,
+ 0.032557252794504166,
+ 0.0588960200548172
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9990143775939941,
+ 0.000985609134659171
+ ],
+ "p_stock_fp16": [
+ 0.9990096092224121,
+ 0.000990307773463428
+ ],
+ "p_fast": [
+ 0.9990096092224121,
+ 0.000990307773463428
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9993529915809631,
+ 0.0006470138905569911
+ ],
+ "p_stock_fp16": [
+ 0.9993494153022766,
+ 0.0006506330100819468
+ ],
+ "p_fast": [
+ 0.9993627667427063,
+ 0.0006372787174768746
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.022473067045211792,
+ 0.3876660168170929,
+ 0.4022129476070404,
+ 0.1876479834318161
+ ],
+ "p_stock_fp16": [
+ 0.02255753055214882,
+ 0.38735076785087585,
+ 0.40278106927871704,
+ 0.1873106062412262
+ ],
+ "p_fast": [
+ 0.02268803119659424,
+ 0.3884519934654236,
+ 0.4015663266181946,
+ 0.18729360401630402
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.031251195818185806,
+ 0.563633143901825,
+ 0.07874128222465515,
+ 0.21205490827560425,
+ 0.05094706267118454,
+ 0.06337244063615799
+ ],
+ "p_stock_fp16": [
+ 0.03140927851200104,
+ 0.5638558864593506,
+ 0.07884733378887177,
+ 0.21141895651817322,
+ 0.05108196288347244,
+ 0.06338655948638916
+ ],
+ "p_fast": [
+ 0.03105660155415535,
+ 0.5640966296195984,
+ 0.0786118432879448,
+ 0.21207797527313232,
+ 0.05080520734190941,
+ 0.06335172057151794
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9974978566169739,
+ 0.0025021692272275686
+ ],
+ "p_stock_fp16": [
+ 0.9974862337112427,
+ 0.0025137581396847963
+ ],
+ "p_fast": [
+ 0.9975180625915527,
+ 0.002481974195688963
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8083035945892334,
+ 0.1916963756084442
+ ],
+ "p_stock_fp16": [
+ 0.8217604756355286,
+ 0.1782395839691162
+ ],
+ "p_fast": [
+ 0.8125700354576111,
+ 0.18742996454238892
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.011750460602343082,
+ 0.46690428256988525,
+ 0.4563930332660675,
+ 0.06495215743780136
+ ],
+ "p_stock_fp16": [
+ 0.011769533157348633,
+ 0.4664737582206726,
+ 0.4565587639808655,
+ 0.06519797444343567
+ ],
+ "p_fast": [
+ 0.011737488210201263,
+ 0.46406930685043335,
+ 0.458662748336792,
+ 0.06553041189908981
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.27544084191322327,
+ 0.14421191811561584,
+ 0.11867069453001022,
+ 0.10384875535964966,
+ 0.055433645844459534,
+ 0.3023941218852997
+ ],
+ "p_stock_fp16": [
+ 0.2752087116241455,
+ 0.14417751133441925,
+ 0.11882904917001724,
+ 0.10405019670724869,
+ 0.05547691881656647,
+ 0.30225759744644165
+ ],
+ "p_fast": [
+ 0.2741827070713043,
+ 0.14392085373401642,
+ 0.11884941160678864,
+ 0.10406802594661713,
+ 0.05548642948269844,
+ 0.30349260568618774
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9962132573127747,
+ 0.003786698682233691
+ ],
+ "p_stock_fp16": [
+ 0.9961954355239868,
+ 0.003804537234827876
+ ],
+ "p_fast": [
+ 0.9961535334587097,
+ 0.0038464043755084276
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9668641090393066,
+ 0.03313582018017769
+ ],
+ "p_stock_fp16": [
+ 0.9673138856887817,
+ 0.03268609941005707
+ ],
+ "p_fast": [
+ 0.9673525094985962,
+ 0.032647524029016495
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.1595691591501236,
+ 0.5837726593017578,
+ 0.2018435150384903,
+ 0.054814662784338
+ ],
+ "p_stock_fp16": [
+ 0.15970534086227417,
+ 0.5836076736450195,
+ 0.20168940722942352,
+ 0.054997652769088745
+ ],
+ "p_fast": [
+ 0.15995338559150696,
+ 0.5836585164070129,
+ 0.20170697569847107,
+ 0.05468110740184784
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.13015204668045044,
+ 0.09355343133211136,
+ 0.21083685755729675,
+ 0.1427493840456009,
+ 0.03295385092496872,
+ 0.3897545039653778
+ ],
+ "p_stock_fp16": [
+ 0.1290270984172821,
+ 0.09375513345003128,
+ 0.21066239476203918,
+ 0.14351902902126312,
+ 0.03291115164756775,
+ 0.39012521505355835
+ ],
+ "p_fast": [
+ 0.12963031232357025,
+ 0.09364315867424011,
+ 0.21082215011119843,
+ 0.14292825758457184,
+ 0.032936107367277145,
+ 0.39003995060920715
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9946586489677429,
+ 0.005341376177966595
+ ],
+ "p_stock_fp16": [
+ 0.994594156742096,
+ 0.005405900068581104
+ ],
+ "p_fast": [
+ 0.9946190118789673,
+ 0.005381016526371241
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9963948130607605,
+ 0.003605250734835863
+ ],
+ "p_stock_fp16": [
+ 0.9963659048080444,
+ 0.003634040243923664
+ ],
+ "p_fast": [
+ 0.9963817596435547,
+ 0.0036181630566716194
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.01470508985221386,
+ 0.11101694405078888,
+ 0.423786461353302,
+ 0.4504915177822113
+ ],
+ "p_stock_fp16": [
+ 0.014744790270924568,
+ 0.11115319281816483,
+ 0.42339763045310974,
+ 0.450704425573349
+ ],
+ "p_fast": [
+ 0.014705481007695198,
+ 0.11096516996622086,
+ 0.4235077500343323,
+ 0.4508216381072998
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.645149290561676,
+ 0.13552716374397278,
+ 0.0501091293990612,
+ 0.03222353383898735,
+ 0.10341624170541763,
+ 0.03357468545436859
+ ],
+ "p_stock_fp16": [
+ 0.6450335383415222,
+ 0.1356692910194397,
+ 0.05025229603052139,
+ 0.03222427889704704,
+ 0.10331260412931442,
+ 0.03350795432925224
+ ],
+ "p_fast": [
+ 0.6442990303039551,
+ 0.1357797384262085,
+ 0.050342343747615814,
+ 0.032313570380210876,
+ 0.10359885543584824,
+ 0.03366648033261299
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.830802857875824,
+ 0.16919711232185364
+ ],
+ "p_stock_fp16": [
+ 0.8311430811882019,
+ 0.16885696351528168
+ ],
+ "p_fast": [
+ 0.8299748301506042,
+ 0.17002512514591217
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9950305223464966,
+ 0.004969438072293997
+ ],
+ "p_stock_fp16": [
+ 0.9950309991836548,
+ 0.004968973807990551
+ ],
+ "p_fast": [
+ 0.9950309991836548,
+ 0.004968973807990551
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.039083082228899,
+ 0.35114800930023193,
+ 0.4543798863887787,
+ 0.1553889960050583
+ ],
+ "p_stock_fp16": [
+ 0.03897734358906746,
+ 0.35115325450897217,
+ 0.4544260799884796,
+ 0.15544334053993225
+ ],
+ "p_fast": [
+ 0.03909020125865936,
+ 0.35113975405693054,
+ 0.45440858602523804,
+ 0.15536148846149445
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.11986440420150757,
+ 0.2974415719509125,
+ 0.13608592748641968,
+ 0.0686950758099556,
+ 0.06451896578073502,
+ 0.31339412927627563
+ ],
+ "p_stock_fp16": [
+ 0.11987487226724625,
+ 0.2972732186317444,
+ 0.1361016035079956,
+ 0.06883835047483444,
+ 0.06454146653413773,
+ 0.3133704662322998
+ ],
+ "p_fast": [
+ 0.1200236901640892,
+ 0.2970615327358246,
+ 0.13600467145442963,
+ 0.0686551034450531,
+ 0.0644955039024353,
+ 0.3137595057487488
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999070167541504,
+ 9.298831719206646e-05
+ ],
+ "p_stock_fp16": [
+ 0.9999096393585205,
+ 9.039064752869308e-05
+ ],
+ "p_fast": [
+ 0.9999082088470459,
+ 9.172435238724574e-05
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9987717270851135,
+ 0.0012282266980037093
+ ],
+ "p_stock_fp16": [
+ 0.9987597465515137,
+ 0.001240298617631197
+ ],
+ "p_fast": [
+ 0.9987645149230957,
+ 0.0012354690115898848
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9965047836303711,
+ 0.0005231389659456909,
+ 0.0006257887580431998,
+ 0.0008441343670710921,
+ 0.0003916886926162988,
+ 0.0011104507138952613
+ ],
+ "p_stock_fp16": [
+ 0.9965031147003174,
+ 0.0005238609155640006,
+ 0.0006257561035454273,
+ 0.0008436930365860462,
+ 0.0003923537442460656,
+ 0.0011111818021163344
+ ],
+ "p_fast": [
+ 0.9964976906776428,
+ 0.0005238580633886158,
+ 0.0006257526692934334,
+ 0.0008469906751997769,
+ 0.0003923516196664423,
+ 0.0011133481748402119
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9498600363731384,
+ 0.050140030682086945
+ ],
+ "p_stock_fp16": [
+ 0.9499313235282898,
+ 0.05006872117519379
+ ],
+ "p_fast": [
+ 0.949651837348938,
+ 0.05034814029932022
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9554705619812012,
+ 0.044529449194669724
+ ],
+ "p_stock_fp16": [
+ 0.9559504985809326,
+ 0.04404947906732559
+ ],
+ "p_fast": [
+ 0.9557444453239441,
+ 0.04425554722547531
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.013081240467727184,
+ 0.543343722820282,
+ 0.44357505440711975
+ ],
+ "p_stock_fp16": [
+ 0.0130953099578619,
+ 0.5433970093727112,
+ 0.44350767135620117
+ ],
+ "p_fast": [
+ 0.01304619014263153,
+ 0.5429470539093018,
+ 0.4440067708492279
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9821338057518005,
+ 0.017866183072328568
+ ],
+ "p_stock_fp16": [
+ 0.9820805788040161,
+ 0.017919491976499557
+ ],
+ "p_fast": [
+ 0.9820826053619385,
+ 0.017917344346642494
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.0011510105105116963,
+ 0.6854820251464844,
+ 0.001569548505358398,
+ 0.30560001730918884,
+ 0.0014406091067939997,
+ 0.0047568148002028465
+ ],
+ "p_stock_fp16": [
+ 0.0011519989930093288,
+ 0.6868552565574646,
+ 0.001574595575220883,
+ 0.30419546365737915,
+ 0.001447755261324346,
+ 0.004774902947247028
+ ],
+ "p_fast": [
+ 0.001150823663920164,
+ 0.6848157048225403,
+ 0.0015729892766103148,
+ 0.30626851320266724,
+ 0.0014406396076083183,
+ 0.004751434549689293
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.381589412689209,
+ 0.618410587310791
+ ],
+ "p_stock_fp16": [
+ 0.38058626651763916,
+ 0.6194137334823608
+ ],
+ "p_fast": [
+ 0.37972337007522583,
+ 0.620276689529419
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7902324795722961,
+ 0.20976750552654266
+ ],
+ "p_stock_fp16": [
+ 0.7897205948829651,
+ 0.21027937531471252
+ ],
+ "p_fast": [
+ 0.7912570834159851,
+ 0.2087429165840149
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.0019981081131845713,
+ 0.1076136901974678,
+ 0.8903881907463074
+ ],
+ "p_stock_fp16": [
+ 0.001993950456380844,
+ 0.10759766399860382,
+ 0.8904083967208862
+ ],
+ "p_fast": [
+ 0.0019904850050807,
+ 0.10741068422794342,
+ 0.8905987739562988
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.983355700969696,
+ 0.01664428971707821
+ ],
+ "p_stock_fp16": [
+ 0.9833749532699585,
+ 0.01662507839500904
+ ],
+ "p_fast": [
+ 0.9833569526672363,
+ 0.01664304919540882
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.05036284402012825,
+ 0.014403803274035454,
+ 0.7842303514480591,
+ 0.05547553300857544,
+ 0.014029274694621563,
+ 0.0814981684088707
+ ],
+ "p_stock_fp16": [
+ 0.05026032403111458,
+ 0.014385768212378025,
+ 0.7846696972846985,
+ 0.055551666766405106,
+ 0.01398407481610775,
+ 0.08114850521087646
+ ],
+ "p_fast": [
+ 0.050325118005275726,
+ 0.014404313638806343,
+ 0.7841482162475586,
+ 0.05555543303489685,
+ 0.014015781693160534,
+ 0.08155122399330139
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9332327246665955,
+ 0.06676729768514633
+ ],
+ "p_stock_fp16": [
+ 0.9334004521369934,
+ 0.0665995329618454
+ ],
+ "p_fast": [
+ 0.9335066080093384,
+ 0.06649336963891983
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9154589772224426,
+ 0.0845409631729126
+ ],
+ "p_stock_fp16": [
+ 0.9153087735176086,
+ 0.08469126373529434
+ ],
+ "p_fast": [
+ 0.9151690602302551,
+ 0.08483093976974487
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.34304532408714294,
+ 0.569991946220398,
+ 0.08696271479129791
+ ],
+ "p_stock_fp16": [
+ 0.3439648151397705,
+ 0.5693216919898987,
+ 0.086713507771492
+ ],
+ "p_fast": [
+ 0.3435242176055908,
+ 0.5697040557861328,
+ 0.08677174896001816
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.1909521371126175,
+ 0.8090478777885437
+ ],
+ "p_stock_fp16": [
+ 0.1911766231060028,
+ 0.8088233470916748
+ ],
+ "p_fast": [
+ 0.19110113382339478,
+ 0.8088988661766052
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.012119858525693417,
+ 0.0884690135717392,
+ 0.012993245385587215,
+ 0.37131616473197937,
+ 0.021717412397265434,
+ 0.49338439106941223
+ ],
+ "p_stock_fp16": [
+ 0.012124559842050076,
+ 0.08864263445138931,
+ 0.013007753528654575,
+ 0.3720622658729553,
+ 0.021741384640336037,
+ 0.49242132902145386
+ ],
+ "p_fast": [
+ 0.012139837257564068,
+ 0.08874349296092987,
+ 0.013011430390179157,
+ 0.37325939536094666,
+ 0.021726302802562714,
+ 0.4911196231842041
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.050892796367406845,
+ 0.9491071701049805
+ ],
+ "p_stock_fp16": [
+ 0.05084657296538353,
+ 0.949153482913971
+ ],
+ "p_fast": [
+ 0.05084657296538353,
+ 0.949153482913971
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.14757339656352997,
+ 0.8524266481399536
+ ],
+ "p_stock_fp16": [
+ 0.14764735102653503,
+ 0.8523526787757874
+ ],
+ "p_fast": [
+ 0.1470339149236679,
+ 0.8529661297798157
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.16986703872680664,
+ 0.4042504131793976,
+ 0.42588260769844055
+ ],
+ "p_stock_fp16": [
+ 0.17007850110530853,
+ 0.4044268727302551,
+ 0.42549458146095276
+ ],
+ "p_fast": [
+ 0.1702163815498352,
+ 0.40435969829559326,
+ 0.42542389035224915
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9974240064620972,
+ 0.0025759965647011995
+ ],
+ "p_stock_fp16": [
+ 0.9974250793457031,
+ 0.0025749404449015856
+ ],
+ "p_fast": [
+ 0.9974281191825867,
+ 0.0025718072429299355
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.007265337277203798,
+ 0.021759064868092537,
+ 0.009209748357534409,
+ 0.04651350528001785,
+ 0.018383000046014786,
+ 0.8968693017959595
+ ],
+ "p_stock_fp16": [
+ 0.007273075170814991,
+ 0.021819675341248512,
+ 0.009230013005435467,
+ 0.04673675075173378,
+ 0.01832025684416294,
+ 0.8966201543807983
+ ],
+ "p_fast": [
+ 0.007309923879802227,
+ 0.02190881408751011,
+ 0.00927677657455206,
+ 0.04715738818049431,
+ 0.018449073657393456,
+ 0.8958980441093445
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.20442381501197815,
+ 0.7955762147903442
+ ],
+ "p_stock_fp16": [
+ 0.2036283314228058,
+ 0.7963716387748718
+ ],
+ "p_fast": [
+ 0.2037075310945511,
+ 0.7962924242019653
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8630937337875366,
+ 0.13690628111362457
+ ],
+ "p_stock_fp16": [
+ 0.8639485836029053,
+ 0.13605144619941711
+ ],
+ "p_fast": [
+ 0.8648214340209961,
+ 0.1351785510778427
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.21081270277500153,
+ 0.6556088328361511,
+ 0.13357853889465332
+ ],
+ "p_stock_fp16": [
+ 0.21070259809494019,
+ 0.6560190320014954,
+ 0.13327835500240326
+ ],
+ "p_fast": [
+ 0.21033410727977753,
+ 0.6555116176605225,
+ 0.13415426015853882
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9561229944229126,
+ 0.043877020478248596
+ ],
+ "p_stock_fp16": [
+ 0.9564744234085083,
+ 0.04352560266852379
+ ],
+ "p_fast": [
+ 0.9564820528030396,
+ 0.04351797699928284
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.0018890078645199537,
+ 0.0025880825705826283,
+ 0.015603619627654552,
+ 0.9757412075996399,
+ 0.000683220278006047,
+ 0.0034948415122926235
+ ],
+ "p_stock_fp16": [
+ 0.0018946017371490598,
+ 0.002579517662525177,
+ 0.015654586255550385,
+ 0.9756941795349121,
+ 0.0006821703282184899,
+ 0.0034949295222759247
+ ],
+ "p_fast": [
+ 0.0018872020300477743,
+ 0.002589595038443804,
+ 0.01564110815525055,
+ 0.9756873846054077,
+ 0.0006861743750050664,
+ 0.0035085841082036495
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 4.781574441947711e-11,
+ 1.0
+ ],
+ "p_stock_fp16": [
+ 4.7722173435182924e-11,
+ 1.0
+ ],
+ "p_fast": [
+ 4.7722173435182924e-11,
+ 1.0
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.001304864534176886,
+ 0.9986951947212219
+ ],
+ "p_stock_fp16": [
+ 0.001314724562689662,
+ 0.9986853003501892
+ ],
+ "p_fast": [
+ 0.0013224405702203512,
+ 0.9986775517463684
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.21309371292591095,
+ 0.5247283577919006,
+ 0.2621779143810272
+ ],
+ "p_stock_fp16": [
+ 0.21319301426410675,
+ 0.5245750546455383,
+ 0.26223188638687134
+ ],
+ "p_fast": [
+ 0.2133568823337555,
+ 0.5244658589363098,
+ 0.2621772885322571
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9018546938896179,
+ 0.09814529865980148
+ ],
+ "p_stock_fp16": [
+ 0.9013143181800842,
+ 0.09868568927049637
+ ],
+ "p_fast": [
+ 0.9016179442405701,
+ 0.09838209301233292
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.01033774483948946,
+ 0.06202789023518562,
+ 0.017014414072036743,
+ 0.7260116338729858,
+ 0.010564091615378857,
+ 0.17404425144195557
+ ],
+ "p_stock_fp16": [
+ 0.010299989953637123,
+ 0.06202596426010132,
+ 0.017031636089086533,
+ 0.727748453617096,
+ 0.01054424699395895,
+ 0.1723497360944748
+ ],
+ "p_fast": [
+ 0.010352902114391327,
+ 0.06196523830294609,
+ 0.017035745084285736,
+ 0.7257944941520691,
+ 0.010598414577543736,
+ 0.17425315082073212
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 3.7185785913607106e-05,
+ 0.9999628067016602
+ ],
+ "p_stock_fp16": [
+ 3.734356869244948e-05,
+ 0.9999626874923706
+ ],
+ "p_fast": [
+ 3.206682958989404e-05,
+ 0.9999679327011108
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.017300518229603767,
+ 0.9826995134353638
+ ],
+ "p_stock_fp16": [
+ 0.01725933328270912,
+ 0.982740581035614
+ ],
+ "p_fast": [
+ 0.017176710069179535,
+ 0.9828232526779175
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.13427391648292542,
+ 0.517401397228241,
+ 0.3483246862888336
+ ],
+ "p_stock_fp16": [
+ 0.13400483131408691,
+ 0.5167117714881897,
+ 0.3492834270000458
+ ],
+ "p_fast": [
+ 0.13415293395519257,
+ 0.5170302987098694,
+ 0.34881681203842163
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.017150158062577248,
+ 0.9828498959541321
+ ],
+ "p_stock_fp16": [
+ 0.01707807369530201,
+ 0.9829219579696655
+ ],
+ "p_fast": [
+ 0.01729249209165573,
+ 0.9827074408531189
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.5351533889770508,
+ 0.045553501695394516,
+ 0.19719760119915009,
+ 0.07893308252096176,
+ 0.02653937041759491,
+ 0.11662296205759048
+ ],
+ "p_stock_fp16": [
+ 0.5362343192100525,
+ 0.04561404883861542,
+ 0.19702892005443573,
+ 0.07889101654291153,
+ 0.026658454909920692,
+ 0.11557310819625854
+ ],
+ "p_fast": [
+ 0.5347406268119812,
+ 0.045866139233112335,
+ 0.19614459574222565,
+ 0.0790177509188652,
+ 0.02664918079972267,
+ 0.11758169531822205
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8605403900146484,
+ 0.13945958018302917
+ ],
+ "p_stock_fp16": [
+ 0.8569320440292358,
+ 0.14306794106960297
+ ],
+ "p_fast": [
+ 0.8601343035697937,
+ 0.13986565172672272
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8890592455863953,
+ 0.11094081401824951
+ ],
+ "p_stock_fp16": [
+ 0.8905324339866638,
+ 0.10946754366159439
+ ],
+ "p_fast": [
+ 0.8911496996879578,
+ 0.10885028541088104
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.17967848479747772,
+ 0.7610151767730713,
+ 0.05930636450648308
+ ],
+ "p_stock_fp16": [
+ 0.17985978722572327,
+ 0.7609444856643677,
+ 0.059195730835199356
+ ],
+ "p_fast": [
+ 0.1796949803829193,
+ 0.7609899640083313,
+ 0.05931501463055611
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.0044141244143247604,
+ 0.9955857992172241
+ ],
+ "p_stock_fp16": [
+ 0.004459704272449017,
+ 0.9955403208732605
+ ],
+ "p_fast": [
+ 0.004382336512207985,
+ 0.9956176280975342
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.18447910249233246,
+ 0.1004968136548996,
+ 0.05253732204437256,
+ 0.46640321612358093,
+ 0.0844254270195961,
+ 0.11165808886289597
+ ],
+ "p_stock_fp16": [
+ 0.18361270427703857,
+ 0.09942716360092163,
+ 0.051961321383714676,
+ 0.4715692698955536,
+ 0.08348112553358078,
+ 0.10994841158390045
+ ],
+ "p_fast": [
+ 0.18718881905078888,
+ 0.10024389624595642,
+ 0.052413731813430786,
+ 0.4646560847759247,
+ 0.0845375582575798,
+ 0.11095986515283585
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.08771830052137375,
+ 0.9122816920280457
+ ],
+ "p_stock_fp16": [
+ 0.08819004893302917,
+ 0.9118099808692932
+ ],
+ "p_fast": [
+ 0.0874079167842865,
+ 0.9125921130180359
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.31802138686180115,
+ 0.6819785833358765
+ ],
+ "p_stock_fp16": [
+ 0.3185911476612091,
+ 0.6814088821411133
+ ],
+ "p_fast": [
+ 0.31933361291885376,
+ 0.6806663870811462
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.011578893288969994,
+ 0.46276065707206726,
+ 0.5256604552268982
+ ],
+ "p_stock_fp16": [
+ 0.01153212133795023,
+ 0.4629037380218506,
+ 0.5255641341209412
+ ],
+ "p_fast": [
+ 0.011598389595746994,
+ 0.46239206194877625,
+ 0.5260095596313477
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8792427778244019,
+ 0.12075725197792053
+ ],
+ "p_stock_fp16": [
+ 0.8800259232521057,
+ 0.11997407674789429
+ ],
+ "p_fast": [
+ 0.880591869354248,
+ 0.11940813809633255
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.03500549495220184,
+ 0.06487958133220673,
+ 0.024736138060688972,
+ 0.2574373483657837,
+ 0.016465064138174057,
+ 0.6014764308929443
+ ],
+ "p_stock_fp16": [
+ 0.03505667671561241,
+ 0.06444761902093887,
+ 0.024641375988721848,
+ 0.25890928506851196,
+ 0.016430731862783432,
+ 0.60051429271698
+ ],
+ "p_fast": [
+ 0.03481389954686165,
+ 0.06428318470716476,
+ 0.02454252727329731,
+ 0.25598886609077454,
+ 0.016396813094615936,
+ 0.6039746999740601
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.0003248288994655013,
+ 0.9996751546859741
+ ],
+ "p_stock_fp16": [
+ 0.000338639976689592,
+ 0.9996613264083862
+ ],
+ "p_fast": [
+ 0.0003301527467556298,
+ 0.9996699094772339
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.07187025249004364,
+ 0.9281297326087952
+ ],
+ "p_stock_fp16": [
+ 0.07211221009492874,
+ 0.9278878569602966
+ ],
+ "p_fast": [
+ 0.07181870192289352,
+ 0.9281812310218811
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.6447005271911621,
+ 0.23754487931728363,
+ 0.11775460094213486
+ ],
+ "p_stock_fp16": [
+ 0.6449388861656189,
+ 0.23772361874580383,
+ 0.11733745783567429
+ ],
+ "p_fast": [
+ 0.6444945335388184,
+ 0.23755982518196106,
+ 0.11794566363096237
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8553566932678223,
+ 0.14464326202869415
+ ],
+ "p_stock_fp16": [
+ 0.8545815348625183,
+ 0.14541849493980408
+ ],
+ "p_fast": [
+ 0.8537299633026123,
+ 0.14627008140087128
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.004985830280929804,
+ 0.9067079424858093,
+ 0.004777889233082533,
+ 0.024800606071949005,
+ 0.005401003174483776,
+ 0.05332663655281067
+ ],
+ "p_stock_fp16": [
+ 0.00499555142596364,
+ 0.9066166281700134,
+ 0.004785445053130388,
+ 0.024830268695950508,
+ 0.005401478614658117,
+ 0.05337066203355789
+ ],
+ "p_fast": [
+ 0.005003904923796654,
+ 0.9063607454299927,
+ 0.004784094635397196,
+ 0.0248232614248991,
+ 0.005421089474111795,
+ 0.05360694229602814
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9907840490341187,
+ 0.009215950965881348
+ ],
+ "p_stock_fp16": [
+ 0.9906796813011169,
+ 0.009320278652012348
+ ],
+ "p_fast": [
+ 0.9907934069633484,
+ 0.009206568822264671
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9906241297721863,
+ 0.00937593262642622
+ ],
+ "p_stock_fp16": [
+ 0.9905851483345032,
+ 0.00941486656665802
+ ],
+ "p_fast": [
+ 0.9906022548675537,
+ 0.009397804737091064
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.0419720783829689,
+ 0.3550425171852112,
+ 0.6029853820800781
+ ],
+ "p_stock_fp16": [
+ 0.042005185037851334,
+ 0.3555469512939453,
+ 0.6024478673934937
+ ],
+ "p_fast": [
+ 0.04203189164400101,
+ 0.35510051250457764,
+ 0.6028676629066467
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5428143739700317,
+ 0.45718565583229065
+ ],
+ "p_stock_fp16": [
+ 0.543347954750061,
+ 0.45665207505226135
+ ],
+ "p_fast": [
+ 0.5426208972930908,
+ 0.4573790729045868
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9188182950019836,
+ 0.016790831461548805,
+ 0.012911411002278328,
+ 0.0055924588814377785,
+ 0.007857374846935272,
+ 0.038029592484235764
+ ],
+ "p_stock_fp16": [
+ 0.9191194176673889,
+ 0.01673590950667858,
+ 0.01286951545625925,
+ 0.005583972670137882,
+ 0.007828657515347004,
+ 0.037862569093704224
+ ],
+ "p_fast": [
+ 0.9193534255027771,
+ 0.01669119857251644,
+ 0.012822604738175869,
+ 0.005574495997279882,
+ 0.007815372198820114,
+ 0.0377429835498333
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 7.057525674980525e-10
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 7.152781145158826e-10
+ ],
+ "p_fast": [
+ 1.0,
+ 7.036732863063833e-10
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999998807907104,
+ 1.521230501566606e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999998807907104,
+ 1.6119919621360168e-07
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 1.5411804099585424e-07
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.29238271713256836,
+ 0.3993513286113739,
+ 0.30826592445373535
+ ],
+ "p_stock_fp16": [
+ 0.2917335331439972,
+ 0.3995320200920105,
+ 0.3087344169616699
+ ],
+ "p_fast": [
+ 0.2921372354030609,
+ 0.3993043005466461,
+ 0.30855846405029297
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9332163333892822,
+ 0.06678367406129837
+ ],
+ "p_stock_fp16": [
+ 0.9334383606910706,
+ 0.06656159460544586
+ ],
+ "p_fast": [
+ 0.9337561726570129,
+ 0.06624376028776169
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/benchmarks/results/parity_multilingual_fp16_rtx4070.json b/benchmarks/results/parity_multilingual_fp16_rtx4070.json
new file mode 100644
index 00000000..59365724
--- /dev/null
+++ b/benchmarks/results/parity_multilingual_fp16_rtx4070.json
@@ -0,0 +1,6122 @@
+{
+ "model": "convaiinnovations/laya",
+ "subfolder": "multilingual",
+ "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER",
+ "torch": "2.11.0+cu130",
+ "dtype": "torch.float16",
+ "summary": {
+ "choice": {
+ "n": 48,
+ "d_fast_stock": 0.006797760725021362,
+ "d_fast_fp32": 0.0018368363380432129,
+ "d_stock_fp32": 0.007908284664154053,
+ "agree_fast_stock": 48,
+ "agree_fast_fp32": 48,
+ "agree_stock_fp32": 48
+ },
+ "noul": {
+ "n": 180,
+ "d_fast_stock": 0.007751792669296265,
+ "d_fast_fp32": 0.008758455514907837,
+ "d_stock_fp32": 0.005219399929046631,
+ "agree_fast_stock": 180,
+ "agree_fast_fp32": 180,
+ "agree_stock_fp32": 180
+ },
+ "score": {
+ "n": 60,
+ "d_fast_stock": 0.001611471176147461,
+ "d_fast_fp32": 0.0008529126644134521,
+ "d_stock_fp32": 0.0012727677822113037,
+ "agree_fast_stock": 60,
+ "agree_fast_fp32": 60,
+ "agree_stock_fp32": 60
+ }
+ },
+ "cases": [
+ {
+ "state": "triage/0",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9947350025177002,
+ 0.00011600032303249463,
+ 0.0005760115454904735,
+ 0.0003717866202350706,
+ 0.0015005050227046013,
+ 0.0027007353492081165
+ ],
+ "p_stock_fp16": [
+ 0.9947320222854614,
+ 0.00011577340046642348,
+ 0.0005833705072291195,
+ 0.0003708128060679883,
+ 0.0014926010044291615,
+ 0.002705391962081194
+ ],
+ "p_fast": [
+ 0.9947184324264526,
+ 0.00011622495367191732,
+ 0.0005913929198868573,
+ 0.00037298683309927583,
+ 0.0014984223525971174,
+ 0.0027027144096791744
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.991271436214447,
+ 0.008728518150746822
+ ],
+ "p_stock_fp16": [
+ 0.9913075566291809,
+ 0.008692430332303047
+ ],
+ "p_fast": [
+ 0.9913222789764404,
+ 0.008677717298269272
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.057799819856882095,
+ 0.34700605273246765,
+ 0.375300794839859,
+ 0.21989338099956512
+ ],
+ "p_stock_fp16": [
+ 0.057760514318943024,
+ 0.34715187549591064,
+ 0.37517744302749634,
+ 0.21991020441055298
+ ],
+ "p_fast": [
+ 0.05788522958755493,
+ 0.3468836545944214,
+ 0.3754371404647827,
+ 0.21979397535324097
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3107343912124634,
+ 0.6892656087875366
+ ],
+ "p_stock_fp16": [
+ 0.31174105405807495,
+ 0.688258945941925
+ ],
+ "p_fast": [
+ 0.3194928467273712,
+ 0.6805071830749512
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.23640592396259308,
+ 0.7635940909385681
+ ],
+ "p_stock_fp16": [
+ 0.23638400435447693,
+ 0.7636159658432007
+ ],
+ "p_fast": [
+ 0.23664849996566772,
+ 0.7633514404296875
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.48604297637939453,
+ 0.5046330094337463,
+ 0.0011995277600362897,
+ 0.0012459942372515798,
+ 0.0026783861685544252,
+ 0.004200144670903683
+ ],
+ "p_stock_fp16": [
+ 0.4781346917152405,
+ 0.5124629735946655,
+ 0.0012085528578609228,
+ 0.0012566962977871299,
+ 0.0027023213915526867,
+ 0.00423477403819561
+ ],
+ "p_fast": [
+ 0.48493245244026184,
+ 0.5057294368743896,
+ 0.0012020273134112358,
+ 0.0012499106815084815,
+ 0.0026824858505278826,
+ 0.0042036897502839565
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8929060101509094,
+ 0.10709398239850998
+ ],
+ "p_stock_fp16": [
+ 0.8927496671676636,
+ 0.10725032538175583
+ ],
+ "p_fast": [
+ 0.8932163119316101,
+ 0.10678370296955109
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.012895478866994381,
+ 0.21030674874782562,
+ 0.45602115988731384,
+ 0.32077664136886597
+ ],
+ "p_stock_fp16": [
+ 0.012891791760921478,
+ 0.21015611290931702,
+ 0.45556217432022095,
+ 0.32138997316360474
+ ],
+ "p_fast": [
+ 0.01289711520075798,
+ 0.21030060946941376,
+ 0.4557502865791321,
+ 0.32105201482772827
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.971129298210144,
+ 0.02887069806456566
+ ],
+ "p_stock_fp16": [
+ 0.9707294702529907,
+ 0.029270583763718605
+ ],
+ "p_fast": [
+ 0.9710330367088318,
+ 0.028966890648007393
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9903473854064941,
+ 0.00965261459350586
+ ],
+ "p_stock_fp16": [
+ 0.9904127717018127,
+ 0.009587177075445652
+ ],
+ "p_fast": [
+ 0.9902961850166321,
+ 0.009703783318400383
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.08989088237285614,
+ 0.0027943237219005823,
+ 0.10268423706293106,
+ 0.22781763970851898,
+ 0.030397633090615273,
+ 0.546415388584137
+ ],
+ "p_stock_fp16": [
+ 0.0898984745144844,
+ 0.0027940424624830484,
+ 0.103472501039505,
+ 0.22738836705684662,
+ 0.030303968116641045,
+ 0.5461426377296448
+ ],
+ "p_fast": [
+ 0.08980085700750351,
+ 0.0027896463871002197,
+ 0.1038660779595375,
+ 0.22625592350959778,
+ 0.030404381453990936,
+ 0.5468831062316895
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9500943422317505,
+ 0.0499056801199913
+ ],
+ "p_stock_fp16": [
+ 0.9501341581344604,
+ 0.049865905195474625
+ ],
+ "p_fast": [
+ 0.9499660730361938,
+ 0.050033897161483765
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.1954011768102646,
+ 0.11548206210136414,
+ 0.15239226818084717,
+ 0.5367245078086853
+ ],
+ "p_stock_fp16": [
+ 0.1953451931476593,
+ 0.11548417806625366,
+ 0.1524324268102646,
+ 0.5367382168769836
+ ],
+ "p_fast": [
+ 0.19591213762760162,
+ 0.11542414873838425,
+ 0.1522044837474823,
+ 0.53645920753479
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9881519675254822,
+ 0.011848055757582188
+ ],
+ "p_stock_fp16": [
+ 0.988207995891571,
+ 0.011792059056460857
+ ],
+ "p_fast": [
+ 0.9880677461624146,
+ 0.011932279914617538
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9542598128318787,
+ 0.045740123838186264
+ ],
+ "p_stock_fp16": [
+ 0.9543293118476868,
+ 0.04567074775695801
+ ],
+ "p_fast": [
+ 0.9541159868240356,
+ 0.04588403180241585
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.0009164155926555395,
+ 0.9796023368835449,
+ 0.0009158870088867843,
+ 0.004492790438234806,
+ 0.005538067780435085,
+ 0.008534470573067665
+ ],
+ "p_stock_fp16": [
+ 0.0009162562200799584,
+ 0.9796056151390076,
+ 0.0009162562200799584,
+ 0.004483622964471579,
+ 0.005536534823477268,
+ 0.008541721850633621
+ ],
+ "p_fast": [
+ 0.0009144751238636672,
+ 0.9796129465103149,
+ 0.0009144751238636672,
+ 0.00450120447203517,
+ 0.005514991004019976,
+ 0.008541785180568695
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9869892001152039,
+ 0.013010749593377113
+ ],
+ "p_stock_fp16": [
+ 0.986968457698822,
+ 0.013031548820436
+ ],
+ "p_fast": [
+ 0.986918032169342,
+ 0.013082008808851242
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.04189760237932205,
+ 0.028212323784828186,
+ 0.23467151820659637,
+ 0.695218563079834
+ ],
+ "p_stock_fp16": [
+ 0.0419837087392807,
+ 0.028241634368896484,
+ 0.2347673922777176,
+ 0.6950072050094604
+ ],
+ "p_fast": [
+ 0.04190762713551521,
+ 0.02824556827545166,
+ 0.23474277555942535,
+ 0.6951040625572205
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999998807907104,
+ 1.161459479703808e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999998807907104,
+ 1.1540090838479955e-07
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 1.1531641064266296e-07
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9712288975715637,
+ 0.028771091252565384
+ ],
+ "p_stock_fp16": [
+ 0.9712384939193726,
+ 0.028761589899659157
+ ],
+ "p_fast": [
+ 0.9711017608642578,
+ 0.028898300603032112
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.16708533465862274,
+ 0.052511539310216904,
+ 0.03815627843141556,
+ 0.05897056311368942,
+ 0.12999477982521057,
+ 0.5532814860343933
+ ],
+ "p_stock_fp16": [
+ 0.16657312214374542,
+ 0.05272258073091507,
+ 0.03827248886227608,
+ 0.059046484529972076,
+ 0.13023501634597778,
+ 0.5531502962112427
+ ],
+ "p_fast": [
+ 0.166844442486763,
+ 0.052705418318510056,
+ 0.03826003149151802,
+ 0.05902726575732231,
+ 0.1301926225423813,
+ 0.5529702305793762
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9995532631874084,
+ 0.0004467671678867191
+ ],
+ "p_stock_fp16": [
+ 0.9995525479316711,
+ 0.00044748239452019334
+ ],
+ "p_fast": [
+ 0.9995502829551697,
+ 0.0004496717010624707
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.02723056450486183,
+ 0.05814352631568909,
+ 0.24995751678943634,
+ 0.6646684408187866
+ ],
+ "p_stock_fp16": [
+ 0.027202146127820015,
+ 0.058152079582214355,
+ 0.24978013336658478,
+ 0.6648656725883484
+ ],
+ "p_fast": [
+ 0.02718338742852211,
+ 0.058111969381570816,
+ 0.2494860142469406,
+ 0.6652186512947083
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999424934387207,
+ 0.0005751202697865665
+ ],
+ "p_stock_fp16": [
+ 0.9994230270385742,
+ 0.0005770346033386886
+ ],
+ "p_fast": [
+ 0.9994257688522339,
+ 0.0005742254434153438
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9941082000732422,
+ 0.005891867447644472
+ ],
+ "p_stock_fp16": [
+ 0.9941064119338989,
+ 0.005893520545214415
+ ],
+ "p_fast": [
+ 0.9941083192825317,
+ 0.0058917333371937275
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.13317157328128815,
+ 0.03420214727520943,
+ 0.03073447197675705,
+ 0.6365819573402405,
+ 0.03473793342709541,
+ 0.1305718868970871
+ ],
+ "p_stock_fp16": [
+ 0.13353483378887177,
+ 0.034160882234573364,
+ 0.030741466209292412,
+ 0.6351954936981201,
+ 0.034902751445770264,
+ 0.13146458566188812
+ ],
+ "p_fast": [
+ 0.13308678567409515,
+ 0.034146152436733246,
+ 0.03078828938305378,
+ 0.6360076069831848,
+ 0.034819621592760086,
+ 0.131151482462883
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8774281740188599,
+ 0.12257183343172073
+ ],
+ "p_stock_fp16": [
+ 0.87716144323349,
+ 0.12283853441476822
+ ],
+ "p_fast": [
+ 0.8775817155838013,
+ 0.12241826206445694
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.1773744821548462,
+ 0.21216727793216705,
+ 0.2800061106681824,
+ 0.3304521143436432
+ ],
+ "p_stock_fp16": [
+ 0.17738482356071472,
+ 0.2123018354177475,
+ 0.2798842787742615,
+ 0.3304290175437927
+ ],
+ "p_fast": [
+ 0.17740105092525482,
+ 0.21190699934959412,
+ 0.2799098789691925,
+ 0.33078211545944214
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.22427187860012054,
+ 0.7757281064987183
+ ],
+ "p_stock_fp16": [
+ 0.22439520061016083,
+ 0.775604784488678
+ ],
+ "p_fast": [
+ 0.22371606528759003,
+ 0.7762839198112488
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.11012163758277893,
+ 0.8898783326148987
+ ],
+ "p_stock_fp16": [
+ 0.10984892398118973,
+ 0.8901510238647461
+ ],
+ "p_fast": [
+ 0.11013572663068771,
+ 0.8898642659187317
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.01340345200151205,
+ 0.7604824304580688,
+ 0.002007364993914962,
+ 0.0050459932535886765,
+ 0.14016902446746826,
+ 0.07889164984226227
+ ],
+ "p_stock_fp16": [
+ 0.013384920544922352,
+ 0.7606458067893982,
+ 0.0020129422191530466,
+ 0.0050506615079939365,
+ 0.1400541514158249,
+ 0.07885149121284485
+ ],
+ "p_fast": [
+ 0.013389640487730503,
+ 0.7594292759895325,
+ 0.0020136521197855473,
+ 0.005052443128079176,
+ 0.14077210426330566,
+ 0.07934282720088959
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9955429434776306,
+ 0.004457075614482164
+ ],
+ "p_stock_fp16": [
+ 0.9954361319541931,
+ 0.004563871305435896
+ ],
+ "p_fast": [
+ 0.9955087900161743,
+ 0.0044912490993738174
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.052439555525779724,
+ 0.2551020383834839,
+ 0.48160114884376526,
+ 0.21085721254348755
+ ],
+ "p_stock_fp16": [
+ 0.052453357726335526,
+ 0.2551765441894531,
+ 0.4816460609436035,
+ 0.21072404086589813
+ ],
+ "p_fast": [
+ 0.052368685603141785,
+ 0.25526267290115356,
+ 0.4815734624862671,
+ 0.21079516410827637
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.6617217823977626e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.6505772748587333e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 1.6485635967455892e-08
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9979941844940186,
+ 0.002005848102271557
+ ],
+ "p_stock_fp16": [
+ 0.9980061650276184,
+ 0.0019938377663493156
+ ],
+ "p_fast": [
+ 0.9979982972145081,
+ 0.0020016254857182503
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.015806416049599648,
+ 0.001576456823386252,
+ 0.16875122487545013,
+ 0.8067996501922607,
+ 0.0019413409754633904,
+ 0.005124932155013084
+ ],
+ "p_stock_fp16": [
+ 0.01580052077770233,
+ 0.001573657151311636,
+ 0.16987161338329315,
+ 0.8056775331497192,
+ 0.0019470063271000981,
+ 0.005129678640514612
+ ],
+ "p_fast": [
+ 0.015823442488908768,
+ 0.0015867498004809022,
+ 0.16945479810237885,
+ 0.8060587048530579,
+ 0.0019441265612840652,
+ 0.00513210566714406
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9989959597587585,
+ 0.00100398575887084
+ ],
+ "p_stock_fp16": [
+ 0.9989975094795227,
+ 0.0010024585062637925
+ ],
+ "p_fast": [
+ 0.998995840549469,
+ 0.001004171441309154
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.12745819985866547,
+ 0.22595053911209106,
+ 0.45733121037483215,
+ 0.18926003575325012
+ ],
+ "p_stock_fp16": [
+ 0.127353698015213,
+ 0.22600941359996796,
+ 0.4578911066055298,
+ 0.18874575197696686
+ ],
+ "p_fast": [
+ 0.12758931517601013,
+ 0.22626180946826935,
+ 0.4572845995426178,
+ 0.18886426091194153
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3306630551815033,
+ 0.6693369746208191
+ ],
+ "p_stock_fp16": [
+ 0.32885265350341797,
+ 0.6711474061012268
+ ],
+ "p_fast": [
+ 0.32739943265914917,
+ 0.6726005673408508
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9962543249130249,
+ 0.003745687659829855
+ ],
+ "p_stock_fp16": [
+ 0.99625164270401,
+ 0.0037483947817236185
+ ],
+ "p_fast": [
+ 0.9962377548217773,
+ 0.0037622100207954645
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.009458568878471851,
+ 0.956800639629364,
+ 0.0030880288686603308,
+ 0.0062720635905861855,
+ 0.006394258700311184,
+ 0.017986396327614784
+ ],
+ "p_stock_fp16": [
+ 0.00940963625907898,
+ 0.9570011496543884,
+ 0.003084840252995491,
+ 0.006243731360882521,
+ 0.006404293701052666,
+ 0.017856350168585777
+ ],
+ "p_fast": [
+ 0.009409425780177116,
+ 0.9569797515869141,
+ 0.003084771102294326,
+ 0.006243591662496328,
+ 0.006391654722392559,
+ 0.01789086125791073
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8893451690673828,
+ 0.11065477877855301
+ ],
+ "p_stock_fp16": [
+ 0.8887225985527039,
+ 0.11127741634845734
+ ],
+ "p_fast": [
+ 0.8888070583343506,
+ 0.11119294166564941
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.011736592277884483,
+ 0.10081744939088821,
+ 0.4652518033981323,
+ 0.42219415307044983
+ ],
+ "p_stock_fp16": [
+ 0.011751952581107616,
+ 0.10092826932668686,
+ 0.4649816155433655,
+ 0.42233821749687195
+ ],
+ "p_fast": [
+ 0.011755738407373428,
+ 0.10086222738027573,
+ 0.4660407602787018,
+ 0.4213412404060364
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9982947707176208,
+ 0.00170520122628659
+ ],
+ "p_stock_fp16": [
+ 0.9983012676239014,
+ 0.0016986510017886758
+ ],
+ "p_fast": [
+ 0.9983129501342773,
+ 0.001687098410911858
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9867876768112183,
+ 0.013212366960942745
+ ],
+ "p_stock_fp16": [
+ 0.98680579662323,
+ 0.013194181025028229
+ ],
+ "p_fast": [
+ 0.9868660569190979,
+ 0.013133921660482883
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.974130392074585,
+ 0.004860663320869207,
+ 0.00267023709602654,
+ 0.0057817562483251095,
+ 0.004317243583500385,
+ 0.00823981873691082
+ ],
+ "p_stock_fp16": [
+ 0.9740860462188721,
+ 0.004872699733823538,
+ 0.002670018933713436,
+ 0.005797786638140678,
+ 0.004316972568631172,
+ 0.008256426081061363
+ ],
+ "p_fast": [
+ 0.9741712212562561,
+ 0.004854126833379269,
+ 0.0026650421787053347,
+ 0.005775688216090202,
+ 0.0043089259415864944,
+ 0.008224956691265106
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999635219573975,
+ 3.64778061339166e-05
+ ],
+ "p_stock_fp16": [
+ 0.9999661445617676,
+ 3.3902360883075744e-05
+ ],
+ "p_fast": [
+ 0.9999663829803467,
+ 3.360159826115705e-05
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.23120898008346558,
+ 0.2528102993965149,
+ 0.2311650663614273,
+ 0.28481560945510864
+ ],
+ "p_stock_fp16": [
+ 0.23089106380939484,
+ 0.25234904885292053,
+ 0.2312295287847519,
+ 0.28553035855293274
+ ],
+ "p_fast": [
+ 0.23083025217056274,
+ 0.2525290846824646,
+ 0.23139449954032898,
+ 0.2852461636066437
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8962268233299255,
+ 0.10377313196659088
+ ],
+ "p_stock_fp16": [
+ 0.8959444761276245,
+ 0.1040554940700531
+ ],
+ "p_fast": [
+ 0.8950363397598267,
+ 0.10496371984481812
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999996423721313,
+ 3.1399471822624037e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999996423721313,
+ 3.1549703294331266e-07
+ ],
+ "p_fast": [
+ 0.9999996423721313,
+ 3.08414342953256e-07
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.0034966091625392437,
+ 0.9029251337051392,
+ 0.0032573991920799017,
+ 0.004987112712115049,
+ 0.005395477171987295,
+ 0.0799381211400032
+ ],
+ "p_stock_fp16": [
+ 0.0035037342458963394,
+ 0.9028777480125427,
+ 0.003265839535742998,
+ 0.00498954625800252,
+ 0.005384459160268307,
+ 0.0799785852432251
+ ],
+ "p_fast": [
+ 0.0034933793358504772,
+ 0.9028506278991699,
+ 0.0032561877742409706,
+ 0.004984525963664055,
+ 0.005400094669312239,
+ 0.08001524209976196
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9991274476051331,
+ 0.00087261653970927
+ ],
+ "p_stock_fp16": [
+ 0.9991249442100525,
+ 0.0008751119603402913
+ ],
+ "p_fast": [
+ 0.9991325736045837,
+ 0.0008674608543515205
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.04123931750655174,
+ 0.2536543905735016,
+ 0.4183993339538574,
+ 0.28670695424079895
+ ],
+ "p_stock_fp16": [
+ 0.04125981032848358,
+ 0.25386014580726624,
+ 0.4183402955532074,
+ 0.2865397334098816
+ ],
+ "p_fast": [
+ 0.041272200644016266,
+ 0.2541844844818115,
+ 0.41805747151374817,
+ 0.286485880613327
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999998807907104,
+ 1.36792962734944e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999998807907104,
+ 1.3650767982653633e-07
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 1.3656601538514224e-07
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9987255930900574,
+ 0.0012743882834911346
+ ],
+ "p_stock_fp16": [
+ 0.9987261891365051,
+ 0.0012738214572891593
+ ],
+ "p_fast": [
+ 0.9987218976020813,
+ 0.0012780847027897835
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.10895277559757233,
+ 0.02216198854148388,
+ 0.02759592980146408,
+ 0.05799749866127968,
+ 0.203551784157753,
+ 0.5797399878501892
+ ],
+ "p_stock_fp16": [
+ 0.10861698538064957,
+ 0.022066883742809296,
+ 0.027462678030133247,
+ 0.057798832654953,
+ 0.20371749997138977,
+ 0.5803371071815491
+ ],
+ "p_fast": [
+ 0.10865631699562073,
+ 0.02211802639067173,
+ 0.027634065598249435,
+ 0.05781975015997887,
+ 0.20379124581813812,
+ 0.5799805521965027
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9987269043922424,
+ 0.0012731034075841308
+ ],
+ "p_stock_fp16": [
+ 0.9987240433692932,
+ 0.001275880727916956
+ ],
+ "p_fast": [
+ 0.998713493347168,
+ 0.0012865019962191582
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.04831191524863243,
+ 0.15560127794742584,
+ 0.4746991991996765,
+ 0.3213876187801361
+ ],
+ "p_stock_fp16": [
+ 0.048204466700553894,
+ 0.15560589730739594,
+ 0.4748738706111908,
+ 0.3213157653808594
+ ],
+ "p_fast": [
+ 0.048220887780189514,
+ 0.15539313852787018,
+ 0.47480377554893494,
+ 0.321582168340683
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999971389770508,
+ 2.8813331027777167e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999971389770508,
+ 2.888175004045479e-06
+ ],
+ "p_fast": [
+ 0.9999971389770508,
+ 2.8885276606160915e-06
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9961869120597839,
+ 0.003813037183135748
+ ],
+ "p_stock_fp16": [
+ 0.9961954355239868,
+ 0.003804537234827876
+ ],
+ "p_fast": [
+ 0.9961982369422913,
+ 0.0038017628248780966
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9980453252792358,
+ 0.0019546218682080507
+ ],
+ "p_stock_fp16": [
+ 0.9980453252792358,
+ 0.0019546342082321644
+ ],
+ "p_fast": [
+ 0.9980443716049194,
+ 0.0019555867183953524
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9945912957191467,
+ 0.005408715922385454
+ ],
+ "p_stock_fp16": [
+ 0.9946163296699524,
+ 0.005383630283176899
+ ],
+ "p_fast": [
+ 0.9945914149284363,
+ 0.005408525932580233
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9982073307037354,
+ 0.001792685710825026
+ ],
+ "p_stock_fp16": [
+ 0.9982055425643921,
+ 0.0017943991115316749
+ ],
+ "p_fast": [
+ 0.9982064962387085,
+ 0.0017935248324647546
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 2.0812866097230653e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 2.0675097189837288e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 2.0675097189837288e-08
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.017536381259560585,
+ 0.28426358103752136,
+ 0.27980342507362366,
+ 0.418396532535553
+ ],
+ "p_stock_fp16": [
+ 0.017525212839245796,
+ 0.2845047414302826,
+ 0.27954739332199097,
+ 0.418422669172287
+ ],
+ "p_fast": [
+ 0.017558783292770386,
+ 0.28477147221565247,
+ 0.27926352620124817,
+ 0.4184061884880066
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9969034790992737,
+ 0.003096469910815358
+ ],
+ "p_stock_fp16": [
+ 0.9969005584716797,
+ 0.0030994152184575796
+ ],
+ "p_fast": [
+ 0.9969141483306885,
+ 0.0030858670361340046
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9887806177139282,
+ 0.011219336651265621
+ ],
+ "p_stock_fp16": [
+ 0.9887717366218567,
+ 0.011228286661207676
+ ],
+ "p_fast": [
+ 0.9887499809265137,
+ 0.011249990202486515
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9887857437133789,
+ 0.011214281432330608
+ ],
+ "p_stock_fp16": [
+ 0.9888203740119934,
+ 0.011179600842297077
+ ],
+ "p_fast": [
+ 0.9888203740119934,
+ 0.011179600842297077
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9993983507156372,
+ 0.0006016937550157309
+ ],
+ "p_stock_fp16": [
+ 0.9993962049484253,
+ 0.0006038257270120084
+ ],
+ "p_fast": [
+ 0.999408483505249,
+ 0.0005915759247727692
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.008935562334954739,
+ 0.125816211104393,
+ 0.338594526052475,
+ 0.5266537070274353
+ ],
+ "p_stock_fp16": [
+ 0.008962121792137623,
+ 0.12574711441993713,
+ 0.3382774591445923,
+ 0.5270133018493652
+ ],
+ "p_fast": [
+ 0.00893578864634037,
+ 0.12545417249202728,
+ 0.33860355615615845,
+ 0.5270064473152161
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9939654469490051,
+ 0.00603454327210784
+ ],
+ "p_stock_fp16": [
+ 0.9939715266227722,
+ 0.006028414238244295
+ ],
+ "p_fast": [
+ 0.9939686059951782,
+ 0.006031340453773737
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9834234714508057,
+ 0.01657659187912941
+ ],
+ "p_stock_fp16": [
+ 0.9834327101707458,
+ 0.016567301005125046
+ ],
+ "p_fast": [
+ 0.983424723148346,
+ 0.01657525822520256
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9984017014503479,
+ 0.0015982583863660693
+ ],
+ "p_stock_fp16": [
+ 0.9983970522880554,
+ 0.0016029166290536523
+ ],
+ "p_fast": [
+ 0.9984087347984314,
+ 0.0015912380767986178
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6506369113922119,
+ 0.3493630886077881
+ ],
+ "p_stock_fp16": [
+ 0.6504672765731812,
+ 0.34953272342681885
+ ],
+ "p_fast": [
+ 0.650245189666748,
+ 0.34975484013557434
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0514373704791069,
+ 0.23009862005710602,
+ 0.39381930232048035,
+ 0.3246447443962097
+ ],
+ "p_stock_fp16": [
+ 0.051297809928655624,
+ 0.23080062866210938,
+ 0.39337265491485596,
+ 0.32452890276908875
+ ],
+ "p_fast": [
+ 0.051470037549734116,
+ 0.23058824241161346,
+ 0.39339467883110046,
+ 0.32454705238342285
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9348726868629456,
+ 0.06512731313705444
+ ],
+ "p_stock_fp16": [
+ 0.9348649978637695,
+ 0.06513497233390808
+ ],
+ "p_fast": [
+ 0.9351320862770081,
+ 0.06486789882183075
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9854146242141724,
+ 0.014585415832698345
+ ],
+ "p_stock_fp16": [
+ 0.9854308366775513,
+ 0.014569154009222984
+ ],
+ "p_fast": [
+ 0.9854275584220886,
+ 0.014572441577911377
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9948474168777466,
+ 0.005152654834091663
+ ],
+ "p_stock_fp16": [
+ 0.994856595993042,
+ 0.005143357440829277
+ ],
+ "p_fast": [
+ 0.9948300719261169,
+ 0.005169972311705351
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8868651390075684,
+ 0.11313491314649582
+ ],
+ "p_stock_fp16": [
+ 0.886721134185791,
+ 0.113278828561306
+ ],
+ "p_fast": [
+ 0.8876070976257324,
+ 0.11239292472600937
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.08663934469223022,
+ 0.3509249985218048,
+ 0.22829334437847137,
+ 0.3341422975063324
+ ],
+ "p_stock_fp16": [
+ 0.08670870959758759,
+ 0.3506438732147217,
+ 0.2282240390777588,
+ 0.33442339301109314
+ ],
+ "p_fast": [
+ 0.08663036674261093,
+ 0.3509262800216675,
+ 0.22824060916900635,
+ 0.3342028260231018
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.16058580577373505,
+ 0.8394142389297485
+ ],
+ "p_stock_fp16": [
+ 0.1605619490146637,
+ 0.8394380211830139
+ ],
+ "p_fast": [
+ 0.1605619490146637,
+ 0.8394380211830139
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.80088871717453,
+ 0.1991112381219864
+ ],
+ "p_stock_fp16": [
+ 0.801042377948761,
+ 0.19895757734775543
+ ],
+ "p_fast": [
+ 0.8005360960960388,
+ 0.1994638741016388
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9967719912528992,
+ 0.0032280399464070797
+ ],
+ "p_stock_fp16": [
+ 0.996775209903717,
+ 0.003224837826564908
+ ],
+ "p_fast": [
+ 0.9967711567878723,
+ 0.003228764282539487
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.959159791469574,
+ 0.040840256959199905
+ ],
+ "p_stock_fp16": [
+ 0.9590489268302917,
+ 0.04095105826854706
+ ],
+ "p_fast": [
+ 0.958798885345459,
+ 0.04120108112692833
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.03479718789458275,
+ 0.5113636255264282,
+ 0.1776311844587326,
+ 0.27620798349380493
+ ],
+ "p_stock_fp16": [
+ 0.03489129617810249,
+ 0.5107249617576599,
+ 0.17805995047092438,
+ 0.2763237953186035
+ ],
+ "p_fast": [
+ 0.03486519306898117,
+ 0.5110909938812256,
+ 0.17792673408985138,
+ 0.27611708641052246
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9932100176811218,
+ 0.00678998650982976
+ ],
+ "p_stock_fp16": [
+ 0.9931843876838684,
+ 0.006815687287598848
+ ],
+ "p_fast": [
+ 0.9932107329368591,
+ 0.00678929453715682
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9863895773887634,
+ 0.013610435649752617
+ ],
+ "p_stock_fp16": [
+ 0.9863746166229248,
+ 0.013625345192849636
+ ],
+ "p_fast": [
+ 0.9863541722297668,
+ 0.013645869679749012
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9996405839920044,
+ 0.00035943291732110083
+ ],
+ "p_stock_fp16": [
+ 0.9996393918991089,
+ 0.00036056042881682515
+ ],
+ "p_fast": [
+ 0.999639630317688,
+ 0.00036038452526554465
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.0008415898191742599,
+ 0.9991583824157715
+ ],
+ "p_stock_fp16": [
+ 0.0008344602538272738,
+ 0.9991655349731445
+ ],
+ "p_fast": [
+ 0.0008295892039313912,
+ 0.9991704225540161
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.08330651372671127,
+ 0.4302324950695038,
+ 0.28755271434783936,
+ 0.1989082545042038
+ ],
+ "p_stock_fp16": [
+ 0.08312342315912247,
+ 0.43046143651008606,
+ 0.28745007514953613,
+ 0.19896505773067474
+ ],
+ "p_fast": [
+ 0.08303479850292206,
+ 0.43000248074531555,
+ 0.2878454625606537,
+ 0.19911719858646393
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9992976188659668,
+ 0.0007023721700534225
+ ],
+ "p_stock_fp16": [
+ 0.9992995262145996,
+ 0.0007004676735959947
+ ],
+ "p_fast": [
+ 0.9992967844009399,
+ 0.0007032073335722089
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999677300453186,
+ 0.00032277024001814425
+ ],
+ "p_stock_fp16": [
+ 0.999679446220398,
+ 0.00032054539769887924
+ ],
+ "p_fast": [
+ 0.999679684638977,
+ 0.0003203890228178352
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9994314312934875,
+ 0.0005685474607162178
+ ],
+ "p_stock_fp16": [
+ 0.9994304776191711,
+ 0.0005694813444279134
+ ],
+ "p_fast": [
+ 0.9994298815727234,
+ 0.0005700374604202807
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999997615814209,
+ 2.3822261141503986e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999997615814209,
+ 2.31881614354279e-07
+ ],
+ "p_fast": [
+ 0.9999997615814209,
+ 2.3589288389302965e-07
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.02595624327659607,
+ 0.23564890027046204,
+ 0.29295405745506287,
+ 0.44544079899787903
+ ],
+ "p_stock_fp16": [
+ 0.025997251272201538,
+ 0.23605045676231384,
+ 0.29262423515319824,
+ 0.4453280568122864
+ ],
+ "p_fast": [
+ 0.025957493111491203,
+ 0.23574696481227875,
+ 0.29310548305511475,
+ 0.44519004225730896
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9991325736045837,
+ 0.0008673956035636365
+ ],
+ "p_stock_fp16": [
+ 0.9991282820701599,
+ 0.0008717031450942159
+ ],
+ "p_fast": [
+ 0.9991348385810852,
+ 0.0008651363314129412
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999171495437622,
+ 8.284277282655239e-05
+ ],
+ "p_stock_fp16": [
+ 0.9999200105667114,
+ 8.002387767191976e-05
+ ],
+ "p_fast": [
+ 0.9999194145202637,
+ 8.053341298364103e-05
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9996435642242432,
+ 0.0003563847276382148
+ ],
+ "p_stock_fp16": [
+ 0.9996417760848999,
+ 0.000358192395651713
+ ],
+ "p_fast": [
+ 0.9996446371078491,
+ 0.00035531914909370244
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 2.18260929329972e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 2.1738845390473216e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 2.1738845390473216e-08
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.06260358542203903,
+ 0.1660652756690979,
+ 0.370134562253952,
+ 0.40119656920433044
+ ],
+ "p_stock_fp16": [
+ 0.06253407895565033,
+ 0.16600701212882996,
+ 0.37010544538497925,
+ 0.40135347843170166
+ ],
+ "p_fast": [
+ 0.0625230222940445,
+ 0.16634276509284973,
+ 0.36994966864585876,
+ 0.4011845588684082
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.970475971698761,
+ 0.029523996636271477
+ ],
+ "p_stock_fp16": [
+ 0.9704890847206116,
+ 0.029510855674743652
+ ],
+ "p_fast": [
+ 0.9704646468162537,
+ 0.029535338282585144
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6521974802017212,
+ 0.3478024899959564
+ ],
+ "p_stock_fp16": [
+ 0.6532374620437622,
+ 0.346762478351593
+ ],
+ "p_fast": [
+ 0.6521306037902832,
+ 0.347869336605072
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9898354411125183,
+ 0.010164561681449413
+ ],
+ "p_stock_fp16": [
+ 0.9898499250411987,
+ 0.010150015354156494
+ ],
+ "p_fast": [
+ 0.9898425936698914,
+ 0.010157376527786255
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.002153037115931511,
+ 0.9978469610214233
+ ],
+ "p_stock_fp16": [
+ 0.002151044551283121,
+ 0.9978489875793457
+ ],
+ "p_fast": [
+ 0.00214894930832088,
+ 0.9978510141372681
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.011194958351552486,
+ 0.5086184144020081,
+ 0.2282281368970871,
+ 0.2519584596157074
+ ],
+ "p_stock_fp16": [
+ 0.011200843378901482,
+ 0.5094704627990723,
+ 0.22777974605560303,
+ 0.2515489459037781
+ ],
+ "p_fast": [
+ 0.011210629716515541,
+ 0.5091691613197327,
+ 0.2277284562587738,
+ 0.25189170241355896
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9984996318817139,
+ 0.0015003288863226771
+ ],
+ "p_stock_fp16": [
+ 0.9985033273696899,
+ 0.0014967061579227448
+ ],
+ "p_fast": [
+ 0.9985026121139526,
+ 0.0014974360819905996
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999946355819702,
+ 5.371579845814267e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999946355819702,
+ 5.399760993896052e-06
+ ],
+ "p_fast": [
+ 0.9999945163726807,
+ 5.478103048517369e-06
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9996912479400635,
+ 0.000308824994135648
+ ],
+ "p_stock_fp16": [
+ 0.999693751335144,
+ 0.0003062819305341691
+ ],
+ "p_fast": [
+ 0.9996925592422485,
+ 0.00030744276591576636
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 5.057518848161635e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 4.925909280473206e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 4.997376734650061e-08
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.12507453560829163,
+ 0.18037156760692596,
+ 0.3176097273826599,
+ 0.3769441843032837
+ ],
+ "p_stock_fp16": [
+ 0.12487392872571945,
+ 0.17992481589317322,
+ 0.3176335394382477,
+ 0.37756773829460144
+ ],
+ "p_fast": [
+ 0.12535706162452698,
+ 0.18018050491809845,
+ 0.31746429204940796,
+ 0.3769981563091278
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9991039633750916,
+ 0.0008960044360719621
+ ],
+ "p_stock_fp16": [
+ 0.9991064667701721,
+ 0.000893554650247097
+ ],
+ "p_fast": [
+ 0.9991050362586975,
+ 0.0008949724142439663
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9994046688079834,
+ 0.0005952878855168819
+ ],
+ "p_stock_fp16": [
+ 0.9994022846221924,
+ 0.0005976693355478346
+ ],
+ "p_fast": [
+ 0.9994021654129028,
+ 0.0005978152039460838
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9996399879455566,
+ 0.00036000681575387716
+ ],
+ "p_stock_fp16": [
+ 0.999640703201294,
+ 0.0003592648427002132
+ ],
+ "p_fast": [
+ 0.999637246131897,
+ 0.0003627449332270771
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 2.6004002506851975e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 2.610388705193145e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 2.6091143467965594e-08
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.031648196280002594,
+ 0.38808199763298035,
+ 0.2868899703025818,
+ 0.2933798134326935
+ ],
+ "p_stock_fp16": [
+ 0.031583767384290695,
+ 0.3883552849292755,
+ 0.2869150638580322,
+ 0.29314592480659485
+ ],
+ "p_fast": [
+ 0.03164965286850929,
+ 0.38878554105758667,
+ 0.28695255517959595,
+ 0.292612224817276
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9997367262840271,
+ 0.0002632086689118296
+ ],
+ "p_stock_fp16": [
+ 0.9997349381446838,
+ 0.0002650434907991439
+ ],
+ "p_fast": [
+ 0.9997336268424988,
+ 0.0002663404738996178
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999908208847046,
+ 9.133191269938834e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999910593032837,
+ 8.957173122325912e-06
+ ],
+ "p_fast": [
+ 0.9999910593032837,
+ 8.887468538887333e-06
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999996423721313,
+ 3.703698041590542e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999996423721313,
+ 3.679090241348604e-07
+ ],
+ "p_fast": [
+ 0.9999996423721313,
+ 3.7079456660649157e-07
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.4538259485163962e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.4560958661036238e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 1.4575186391141415e-08
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.020183581858873367,
+ 0.24479416012763977,
+ 0.35684293508529663,
+ 0.3781793415546417
+ ],
+ "p_stock_fp16": [
+ 0.020182596519589424,
+ 0.24455727636814117,
+ 0.35722142457962036,
+ 0.3780386447906494
+ ],
+ "p_fast": [
+ 0.02019709162414074,
+ 0.24473293125629425,
+ 0.3571290969848633,
+ 0.37794092297554016
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999994039535522,
+ 6.382118158398953e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999992847442627,
+ 6.629520612477791e-07
+ ],
+ "p_fast": [
+ 0.9999994039535522,
+ 6.477529836956819e-07
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 3.7367424710055275e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 3.752927923983407e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 3.7538438135698016e-08
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9997164607048035,
+ 0.00028347765328362584
+ ],
+ "p_stock_fp16": [
+ 0.9997203946113586,
+ 0.00027952788514085114
+ ],
+ "p_fast": [
+ 0.9997139573097229,
+ 0.00028601515805348754
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0636468306183815,
+ 0.22837206721305847,
+ 0.21096466481685638,
+ 0.49701645970344543
+ ],
+ "p_stock_fp16": [
+ 0.06364992260932922,
+ 0.22798435389995575,
+ 0.21064522862434387,
+ 0.49772047996520996
+ ],
+ "p_fast": [
+ 0.06365229189395905,
+ 0.22799281775951385,
+ 0.21085889637470245,
+ 0.4974960386753082
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.8883478045463562,
+ 0.008352874778211117,
+ 0.0076321023516356945,
+ 0.01150154136121273,
+ 0.001612467342056334,
+ 0.08255330473184586
+ ],
+ "p_stock_fp16": [
+ 0.8884479403495789,
+ 0.00837636087089777,
+ 0.0076118819415569305,
+ 0.011482718400657177,
+ 0.0016143446555361152,
+ 0.08246690034866333
+ ],
+ "p_fast": [
+ 0.8885501027107239,
+ 0.008344664238393307,
+ 0.007612757384777069,
+ 0.011461631394922733,
+ 0.0016082359943538904,
+ 0.08242260664701462
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999997615814209,
+ 2.4383007257711142e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999974966049194,
+ 2.4849055080267135e-06
+ ],
+ "p_fast": [
+ 0.9999974966049194,
+ 2.50439484261733e-06
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9966451525688171,
+ 0.003354875138029456
+ ],
+ "p_stock_fp16": [
+ 0.9966463446617126,
+ 0.003353686071932316
+ ],
+ "p_fast": [
+ 0.9966447353363037,
+ 0.003355318447574973
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9906609654426575,
+ 0.009339054115116596
+ ],
+ "p_stock_fp16": [
+ 0.9907005429267883,
+ 0.009299449622631073
+ ],
+ "p_fast": [
+ 0.9907229542732239,
+ 0.009276984259486198
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0063354335725307465,
+ 0.06066763401031494,
+ 0.13907092809677124,
+ 0.7939260005950928
+ ],
+ "p_stock_fp16": [
+ 0.006317968014627695,
+ 0.06053146347403526,
+ 0.13891354203224182,
+ 0.7942370772361755
+ ],
+ "p_fast": [
+ 0.006321958731859922,
+ 0.060628872364759445,
+ 0.13908614218235016,
+ 0.7939630150794983
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.8043204545974731,
+ 0.11213378608226776,
+ 0.011883067898452282,
+ 0.006389721762388945,
+ 0.017634831368923187,
+ 0.04763811454176903
+ ],
+ "p_stock_fp16": [
+ 0.8046354055404663,
+ 0.11216042190790176,
+ 0.0117985550314188,
+ 0.006340028718113899,
+ 0.017573898658156395,
+ 0.04749171435832977
+ ],
+ "p_fast": [
+ 0.8043616414070129,
+ 0.11214965581893921,
+ 0.011866750195622444,
+ 0.006376673933118582,
+ 0.017572207376360893,
+ 0.04767300933599472
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9990018010139465,
+ 0.0009981930488720536
+ ],
+ "p_stock_fp16": [
+ 0.9989840388298035,
+ 0.0010159964440390468
+ ],
+ "p_fast": [
+ 0.9990046620368958,
+ 0.000995393143966794
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999978542327881,
+ 2.116629730153363e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999978542327881,
+ 2.1285686671035364e-06
+ ],
+ "p_fast": [
+ 0.9999978542327881,
+ 2.0945451524312375e-06
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9922552108764648,
+ 0.00774480402469635
+ ],
+ "p_stock_fp16": [
+ 0.9922477006912231,
+ 0.0077522494830191135
+ ],
+ "p_fast": [
+ 0.9922411441802979,
+ 0.0077588255517184734
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.22696898877620697,
+ 0.49794095754623413,
+ 0.17827990651130676,
+ 0.09681016951799393
+ ],
+ "p_stock_fp16": [
+ 0.22703060507774353,
+ 0.49782127141952515,
+ 0.178546741604805,
+ 0.09660134464502335
+ ],
+ "p_fast": [
+ 0.22649630904197693,
+ 0.4985935389995575,
+ 0.17830058932304382,
+ 0.09660957753658295
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.055354341864585876,
+ 0.004938972182571888,
+ 0.028731388971209526,
+ 0.2593582570552826,
+ 0.014712448231875896,
+ 0.6369046568870544
+ ],
+ "p_stock_fp16": [
+ 0.05537652224302292,
+ 0.004957123193889856,
+ 0.028721945360302925,
+ 0.25951990485191345,
+ 0.014727150090038776,
+ 0.6366972923278809
+ ],
+ "p_fast": [
+ 0.05525214225053787,
+ 0.004929112736135721,
+ 0.028727484866976738,
+ 0.2595699727535248,
+ 0.014701249077916145,
+ 0.6368200778961182
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.004081618506461382,
+ 0.9959184527397156
+ ],
+ "p_stock_fp16": [
+ 0.004087990149855614,
+ 0.9959120750427246
+ ],
+ "p_fast": [
+ 0.004082030616700649,
+ 0.9959179759025574
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.004883488640189171,
+ 0.9951164722442627
+ ],
+ "p_stock_fp16": [
+ 0.004874517675489187,
+ 0.9951254725456238
+ ],
+ "p_fast": [
+ 0.004876886494457722,
+ 0.9951230883598328
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9911409020423889,
+ 0.00885908305644989
+ ],
+ "p_stock_fp16": [
+ 0.9911389946937561,
+ 0.008860963396728039
+ ],
+ "p_fast": [
+ 0.9911715388298035,
+ 0.008828491903841496
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0812956839799881,
+ 0.23141789436340332,
+ 0.5429924130439758,
+ 0.14429405331611633
+ ],
+ "p_stock_fp16": [
+ 0.08134075254201889,
+ 0.2311532199382782,
+ 0.5428540706634521,
+ 0.14465197920799255
+ ],
+ "p_fast": [
+ 0.08118554949760437,
+ 0.23116320371627808,
+ 0.5432752966880798,
+ 0.1443759799003601
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.004492489621043205,
+ 0.9707549214363098,
+ 0.0053355759009718895,
+ 0.01640835963189602,
+ 0.0014624387258663774,
+ 0.001546246581710875
+ ],
+ "p_stock_fp16": [
+ 0.004491300322115421,
+ 0.9707983136177063,
+ 0.005317947827279568,
+ 0.016382431611418724,
+ 0.0014623894821852446,
+ 0.0015476110856980085
+ ],
+ "p_fast": [
+ 0.004503846634179354,
+ 0.9706624150276184,
+ 0.005338014103472233,
+ 0.01647438481450081,
+ 0.001467907684855163,
+ 0.0015534505946561694
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3688707947731018,
+ 0.6311292052268982
+ ],
+ "p_stock_fp16": [
+ 0.37409019470214844,
+ 0.6259098052978516
+ ],
+ "p_fast": [
+ 0.3744475245475769,
+ 0.6255525350570679
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9971032738685608,
+ 0.002896667458117008
+ ],
+ "p_stock_fp16": [
+ 0.997104823589325,
+ 0.0028952136635780334
+ ],
+ "p_fast": [
+ 0.9970920085906982,
+ 0.0029079276137053967
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9995230436325073,
+ 0.00047693666419945657
+ ],
+ "p_stock_fp16": [
+ 0.9995192289352417,
+ 0.0004807662917301059
+ ],
+ "p_fast": [
+ 0.9995251893997192,
+ 0.00047482014633715153
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.05629942938685417,
+ 0.2538512647151947,
+ 0.5029294490814209,
+ 0.1869198977947235
+ ],
+ "p_stock_fp16": [
+ 0.0563054084777832,
+ 0.25407424569129944,
+ 0.5028257369995117,
+ 0.18679453432559967
+ ],
+ "p_fast": [
+ 0.056361015886068344,
+ 0.25382891297340393,
+ 0.5028310418128967,
+ 0.1869790107011795
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.04281329736113548,
+ 0.15090227127075195,
+ 0.31571245193481445,
+ 0.10606011003255844,
+ 0.03266778960824013,
+ 0.35184410214424133
+ ],
+ "p_stock_fp16": [
+ 0.04281585291028023,
+ 0.15120358765125275,
+ 0.31575140357017517,
+ 0.10597025603055954,
+ 0.0327000692486763,
+ 0.3515588641166687
+ ],
+ "p_fast": [
+ 0.04285489767789841,
+ 0.15104618668556213,
+ 0.31542274355888367,
+ 0.10606689751148224,
+ 0.03272989019751549,
+ 0.3518794775009155
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.0005439565284177661,
+ 0.999455988407135
+ ],
+ "p_stock_fp16": [
+ 0.000552778656128794,
+ 0.9994471669197083
+ ],
+ "p_fast": [
+ 0.0005198119324631989,
+ 0.9994801878929138
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 2.2439931854023598e-05,
+ 0.9999775886535645
+ ],
+ "p_stock_fp16": [
+ 2.2962143702898175e-05,
+ 0.9999769926071167
+ ],
+ "p_fast": [
+ 2.2496045858133584e-05,
+ 0.9999774694442749
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5837574601173401,
+ 0.4162425398826599
+ ],
+ "p_stock_fp16": [
+ 0.5840339660644531,
+ 0.4159660339355469
+ ],
+ "p_fast": [
+ 0.5823723077774048,
+ 0.4176277220249176
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.19197113811969757,
+ 0.3132495880126953,
+ 0.36341583728790283,
+ 0.1313633769750595
+ ],
+ "p_stock_fp16": [
+ 0.19125224649906158,
+ 0.31317344307899475,
+ 0.36406445503234863,
+ 0.13150981068611145
+ ],
+ "p_fast": [
+ 0.19186653196811676,
+ 0.3130118250846863,
+ 0.36383211612701416,
+ 0.13128958642482758
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.8871828317642212,
+ 0.03427496179938316,
+ 0.007215311285108328,
+ 0.03236454725265503,
+ 0.009683055803179741,
+ 0.02927929349243641
+ ],
+ "p_stock_fp16": [
+ 0.8875331282615662,
+ 0.03414558619260788,
+ 0.0071993637830019,
+ 0.03231260925531387,
+ 0.009603030048310757,
+ 0.02920626848936081
+ ],
+ "p_fast": [
+ 0.8873127698898315,
+ 0.034203849732875824,
+ 0.007211647927761078,
+ 0.032273054122924805,
+ 0.009685399010777473,
+ 0.02931329980492592
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999998807907104,
+ 1.6774728806012718e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999998807907104,
+ 1.6465972407786467e-07
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 1.6478036002354202e-07
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999998807907104,
+ 6.427991650070908e-08
+ ],
+ "p_stock_fp16": [
+ 0.9999998807907104,
+ 6.473403857398807e-08
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 6.369931782046478e-08
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9454833269119263,
+ 0.05451667308807373
+ ],
+ "p_stock_fp16": [
+ 0.9454497694969177,
+ 0.05455021560192108
+ ],
+ "p_fast": [
+ 0.9455316066741943,
+ 0.05446843057870865
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.09172669053077698,
+ 0.3128417730331421,
+ 0.4629928469657898,
+ 0.13243865966796875
+ ],
+ "p_stock_fp16": [
+ 0.09185315668582916,
+ 0.31290489435195923,
+ 0.4628952443599701,
+ 0.1323467642068863
+ ],
+ "p_fast": [
+ 0.09187891334295273,
+ 0.31253448128700256,
+ 0.46325117349624634,
+ 0.13233540952205658
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.22288650274276733,
+ 0.7018542289733887,
+ 0.006291983649134636,
+ 0.0043691350147128105,
+ 0.055768974125385284,
+ 0.008829258382320404
+ ],
+ "p_stock_fp16": [
+ 0.22255434095859528,
+ 0.702457845211029,
+ 0.0062703704461455345,
+ 0.004360357765108347,
+ 0.05554034933447838,
+ 0.00881677120923996
+ ],
+ "p_fast": [
+ 0.2229149341583252,
+ 0.7022231221199036,
+ 0.006292808335274458,
+ 0.00436742277815938,
+ 0.05539654940366745,
+ 0.008805222809314728
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999981164932251,
+ 1.8847378669306636e-05
+ ],
+ "p_stock_fp16": [
+ 0.9999806880950928,
+ 1.9359751604497433e-05
+ ],
+ "p_fast": [
+ 0.9999808073043823,
+ 1.9209097445127554e-05
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999991655349731,
+ 8.589877893427911e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999991655349731,
+ 8.657084435981233e-07
+ ],
+ "p_fast": [
+ 0.9999991655349731,
+ 8.596007319283672e-07
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9994506239891052,
+ 0.0005493529606610537
+ ],
+ "p_stock_fp16": [
+ 0.9994482398033142,
+ 0.000551776378415525
+ ],
+ "p_fast": [
+ 0.9994507431983948,
+ 0.0005492451600730419
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.14145158231258392,
+ 0.5174408555030823,
+ 0.22789683938026428,
+ 0.11321073770523071
+ ],
+ "p_stock_fp16": [
+ 0.14142026007175446,
+ 0.5172938108444214,
+ 0.22798392176628113,
+ 0.11330195516347885
+ ],
+ "p_fast": [
+ 0.1412927210330963,
+ 0.5178377628326416,
+ 0.22800087928771973,
+ 0.11286862194538116
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.2503761649131775,
+ 0.4258953630924225,
+ 0.21602272987365723,
+ 0.06063825264573097,
+ 0.01531359925866127,
+ 0.03175388649106026
+ ],
+ "p_stock_fp16": [
+ 0.24969635903835297,
+ 0.4277918338775635,
+ 0.2153831422328949,
+ 0.06020161882042885,
+ 0.015295857563614845,
+ 0.0316312201321125
+ ],
+ "p_fast": [
+ 0.25056368112564087,
+ 0.4259240925312042,
+ 0.21582148969173431,
+ 0.060586120933294296,
+ 0.015333536081016064,
+ 0.031771134585142136
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999997615814209,
+ 1.8002197066380177e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999997615814209,
+ 1.8549380342847144e-07
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 1.7669721330548782e-07
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9992652535438538,
+ 0.0007346862112171948
+ ],
+ "p_stock_fp16": [
+ 0.9992583394050598,
+ 0.000741687195841223
+ ],
+ "p_fast": [
+ 0.9992583394050598,
+ 0.0007415966247208416
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999640941619873,
+ 0.000359025172656402
+ ],
+ "p_stock_fp16": [
+ 0.9996439218521118,
+ 0.0003561005287338048
+ ],
+ "p_fast": [
+ 0.9996427297592163,
+ 0.000357319280738011
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0687880590558052,
+ 0.25616005063056946,
+ 0.4207719564437866,
+ 0.2542799413204193
+ ],
+ "p_stock_fp16": [
+ 0.06884820759296417,
+ 0.25655263662338257,
+ 0.42159199714660645,
+ 0.253007173538208
+ ],
+ "p_fast": [
+ 0.06868042796850204,
+ 0.2558024823665619,
+ 0.4208984076976776,
+ 0.25461864471435547
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.4140915274620056,
+ 0.029255375266075134,
+ 0.08089007437229156,
+ 0.07105124741792679,
+ 0.0144424419850111,
+ 0.39026930928230286
+ ],
+ "p_stock_fp16": [
+ 0.4145292043685913,
+ 0.029076112434267998,
+ 0.08059592545032501,
+ 0.07084836810827255,
+ 0.014393718913197517,
+ 0.390556663274765
+ ],
+ "p_fast": [
+ 0.413266122341156,
+ 0.02930057980120182,
+ 0.08105966448783875,
+ 0.07097821682691574,
+ 0.014504841528832912,
+ 0.39089056849479675
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999998807907104,
+ 7.432532811435522e-08
+ ],
+ "p_stock_fp16": [
+ 0.9999998807907104,
+ 7.468143792266346e-08
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 7.474527308204415e-08
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999998807907104,
+ 1.072692157322308e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999998807907104,
+ 1.0675445594188204e-07
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 1.0691094587400585e-07
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.7127511853232136e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.7115166173198304e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 1.7073434221970274e-08
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.16459648311138153,
+ 0.21402424573898315,
+ 0.2646333575248718,
+ 0.3567458987236023
+ ],
+ "p_stock_fp16": [
+ 0.16502077877521515,
+ 0.21399639546871185,
+ 0.2649608850479126,
+ 0.3560219407081604
+ ],
+ "p_fast": [
+ 0.1650002896785736,
+ 0.21383924782276154,
+ 0.2644432783126831,
+ 0.35671716928482056
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.39744681119918823,
+ 0.0681687667965889,
+ 0.09657040983438492,
+ 0.12045320123434067,
+ 0.04587395489215851,
+ 0.2714868485927582
+ ],
+ "p_stock_fp16": [
+ 0.3964879512786865,
+ 0.06836309283971786,
+ 0.0965958908200264,
+ 0.12068604677915573,
+ 0.04589680954813957,
+ 0.2719702124595642
+ ],
+ "p_fast": [
+ 0.3966870605945587,
+ 0.06813076883554459,
+ 0.09664440155029297,
+ 0.12051105499267578,
+ 0.045919857919216156,
+ 0.272106796503067
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9837604761123657,
+ 0.016239557415246964
+ ],
+ "p_stock_fp16": [
+ 0.9837303161621094,
+ 0.016269642859697342
+ ],
+ "p_fast": [
+ 0.9835969805717468,
+ 0.01640303246676922
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9866739511489868,
+ 0.013326093554496765
+ ],
+ "p_stock_fp16": [
+ 0.9866234660148621,
+ 0.013376603834331036
+ ],
+ "p_fast": [
+ 0.986610472202301,
+ 0.013389497064054012
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9993465542793274,
+ 0.0006534745334647596
+ ],
+ "p_stock_fp16": [
+ 0.9993439316749573,
+ 0.0006560726324096322
+ ],
+ "p_fast": [
+ 0.9993391633033752,
+ 0.0006608318653889
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.029554622247815132,
+ 0.5994492173194885,
+ 0.3388322591781616,
+ 0.032163914293050766
+ ],
+ "p_stock_fp16": [
+ 0.02967812493443489,
+ 0.5990188121795654,
+ 0.338985413312912,
+ 0.03231770917773247
+ ],
+ "p_fast": [
+ 0.02963946759700775,
+ 0.5991156101226807,
+ 0.3390401601791382,
+ 0.032204773277044296
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.055205926299095154,
+ 0.8407003283500671,
+ 0.015516695566475391,
+ 0.0042560226283967495,
+ 0.006794240791350603,
+ 0.07752686738967896
+ ],
+ "p_stock_fp16": [
+ 0.055257003754377365,
+ 0.8402292728424072,
+ 0.015570740215480328,
+ 0.0042651258409023285,
+ 0.006828976329416037,
+ 0.07784880697727203
+ ],
+ "p_fast": [
+ 0.05513295531272888,
+ 0.8408026099205017,
+ 0.015550963580608368,
+ 0.00425970833748579,
+ 0.006806994788348675,
+ 0.07744680345058441
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9939618706703186,
+ 0.0060381051152944565
+ ],
+ "p_stock_fp16": [
+ 0.9940674304962158,
+ 0.005932627245783806
+ ],
+ "p_fast": [
+ 0.9939715266227722,
+ 0.006028414238244295
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999995231628418,
+ 4.784168368132669e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999995231628418,
+ 4.913429165753769e-07
+ ],
+ "p_fast": [
+ 0.9999995231628418,
+ 4.4945284116693074e-07
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9998462200164795,
+ 0.00015373615315183997
+ ],
+ "p_stock_fp16": [
+ 0.9998452663421631,
+ 0.00015465331671293825
+ ],
+ "p_fast": [
+ 0.9998350143432617,
+ 0.00016494776355102658
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.06287779659032822,
+ 0.5351999998092651,
+ 0.3278793394565582,
+ 0.07404284924268723
+ ],
+ "p_stock_fp16": [
+ 0.06269378215074539,
+ 0.5347920656204224,
+ 0.3285119831562042,
+ 0.07400219887495041
+ ],
+ "p_fast": [
+ 0.06284317374229431,
+ 0.534889817237854,
+ 0.32825133204460144,
+ 0.07401572167873383
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.2567249536514282,
+ 0.2075771987438202,
+ 0.04681433364748955,
+ 0.0250278040766716,
+ 0.0229067150503397,
+ 0.4409489631652832
+ ],
+ "p_stock_fp16": [
+ 0.2571020722389221,
+ 0.20699118077754974,
+ 0.0467303991317749,
+ 0.02506188303232193,
+ 0.022908415645360947,
+ 0.4412061274051666
+ ],
+ "p_fast": [
+ 0.25635719299316406,
+ 0.2071993201971054,
+ 0.04686884582042694,
+ 0.025038132444024086,
+ 0.02288670651614666,
+ 0.44164976477622986
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.020514102652668953,
+ 0.3828057050704956,
+ 0.32532671093940735,
+ 0.27135351300239563
+ ],
+ "p_stock_fp16": [
+ 0.020666882395744324,
+ 0.3833477795124054,
+ 0.3253430128097534,
+ 0.27064231038093567
+ ],
+ "p_fast": [
+ 0.02068278193473816,
+ 0.38345545530319214,
+ 0.3252755403518677,
+ 0.27058616280555725
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.061993859708309174,
+ 0.018191542476415634,
+ 0.38648930191993713,
+ 0.08014030754566193,
+ 0.009092590771615505,
+ 0.44409245252609253
+ ],
+ "p_stock_fp16": [
+ 0.06193697452545166,
+ 0.018130609765648842,
+ 0.3857613205909729,
+ 0.0799960047006607,
+ 0.0090811001136899,
+ 0.44509389996528625
+ ],
+ "p_fast": [
+ 0.06192393600940704,
+ 0.018162235617637634,
+ 0.3870006501674652,
+ 0.0801355391740799,
+ 0.009079190902411938,
+ 0.4436984360218048
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9922048449516296,
+ 0.007795148994773626
+ ],
+ "p_stock_fp16": [
+ 0.9922584891319275,
+ 0.007741458248347044
+ ],
+ "p_fast": [
+ 0.9922322034835815,
+ 0.007767758332192898
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.678847074508667,
+ 0.321152925491333
+ ],
+ "p_stock_fp16": [
+ 0.6824679374694824,
+ 0.3175320625305176
+ ],
+ "p_fast": [
+ 0.6786465048789978,
+ 0.3213534951210022
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.006523916497826576,
+ 0.6323118209838867,
+ 0.262757271528244,
+ 0.09840697795152664
+ ],
+ "p_stock_fp16": [
+ 0.006531151942908764,
+ 0.6330553889274597,
+ 0.2621628940105438,
+ 0.09825055301189423
+ ],
+ "p_fast": [
+ 0.006546929944306612,
+ 0.632265031337738,
+ 0.2627962529659271,
+ 0.09839177131652832
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9610563516616821,
+ 0.012892201542854309,
+ 0.006290849763900042,
+ 0.004069325514137745,
+ 0.005694213788956404,
+ 0.009997155517339706
+ ],
+ "p_stock_fp16": [
+ 0.9613276124000549,
+ 0.012819050811231136,
+ 0.006247506942600012,
+ 0.004041579551994801,
+ 0.005600228905677795,
+ 0.009964006021618843
+ ],
+ "p_fast": [
+ 0.9610597491264343,
+ 0.012903384864330292,
+ 0.006288608070462942,
+ 0.004068167880177498,
+ 0.005670197773724794,
+ 0.010009989142417908
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9047498106956482,
+ 0.09525013715028763
+ ],
+ "p_stock_fp16": [
+ 0.9049659371376038,
+ 0.09503404796123505
+ ],
+ "p_fast": [
+ 0.9056148529052734,
+ 0.09438513219356537
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.904305100440979,
+ 0.09569495171308517
+ ],
+ "p_stock_fp16": [
+ 0.9042285680770874,
+ 0.09577147662639618
+ ],
+ "p_fast": [
+ 0.9041438698768616,
+ 0.09585607796907425
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.3857632577419281,
+ 0.21712790429592133,
+ 0.33055809140205383,
+ 0.06655073165893555
+ ],
+ "p_stock_fp16": [
+ 0.38597556948661804,
+ 0.2175724357366562,
+ 0.3298201560974121,
+ 0.06663182377815247
+ ],
+ "p_fast": [
+ 0.385956734418869,
+ 0.21756181120872498,
+ 0.32980403304100037,
+ 0.06667738407850266
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.012203803285956383,
+ 0.0018551512621343136,
+ 0.06903497874736786,
+ 0.006404262501746416,
+ 0.0010881084017455578,
+ 0.9094136357307434
+ ],
+ "p_stock_fp16": [
+ 0.012242876924574375,
+ 0.0018610768020153046,
+ 0.06918726861476898,
+ 0.00642011733725667,
+ 0.001091935788281262,
+ 0.9091967344284058
+ ],
+ "p_fast": [
+ 0.012145044282078743,
+ 0.0018552416004240513,
+ 0.0686427652835846,
+ 0.006375036668032408,
+ 0.0010885121300816536,
+ 0.9098933339118958
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9826712012290955,
+ 0.01732879877090454
+ ],
+ "p_stock_fp16": [
+ 0.9826159477233887,
+ 0.0173840019851923
+ ],
+ "p_fast": [
+ 0.9827365875244141,
+ 0.01726347580552101
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9772809147834778,
+ 0.022719144821166992
+ ],
+ "p_stock_fp16": [
+ 0.9771429300308228,
+ 0.022857103496789932
+ ],
+ "p_fast": [
+ 0.9772191047668457,
+ 0.0227808840572834
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.03583744168281555,
+ 0.8142819404602051,
+ 0.10607772320508957,
+ 0.04380291700363159
+ ],
+ "p_stock_fp16": [
+ 0.03587322682142258,
+ 0.8140823245048523,
+ 0.10613458603620529,
+ 0.043909892439842224
+ ],
+ "p_fast": [
+ 0.03581710532307625,
+ 0.8143978118896484,
+ 0.1060720831155777,
+ 0.04371294379234314
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.5059749484062195,
+ 0.03354005515575409,
+ 0.3780619502067566,
+ 0.025879623368382454,
+ 0.00476288003847003,
+ 0.05178051069378853
+ ],
+ "p_stock_fp16": [
+ 0.5070874691009521,
+ 0.03352575749158859,
+ 0.37727269530296326,
+ 0.025755342096090317,
+ 0.00473642535507679,
+ 0.05162234231829643
+ ],
+ "p_fast": [
+ 0.5062479972839355,
+ 0.033590998500585556,
+ 0.3778684735298157,
+ 0.025805464014410973,
+ 0.00476421695202589,
+ 0.051722802221775055
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9258522987365723,
+ 0.07414764910936356
+ ],
+ "p_stock_fp16": [
+ 0.9258021116256714,
+ 0.07419782131910324
+ ],
+ "p_fast": [
+ 0.9258357286453247,
+ 0.07416428625583649
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9257611632347107,
+ 0.07423881441354752
+ ],
+ "p_stock_fp16": [
+ 0.9255081415176392,
+ 0.07449184358119965
+ ],
+ "p_fast": [
+ 0.925844132900238,
+ 0.07415590435266495
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.020395394414663315,
+ 0.8977064490318298,
+ 0.06564648449420929,
+ 0.016251610592007637
+ ],
+ "p_stock_fp16": [
+ 0.02042355015873909,
+ 0.8977500796318054,
+ 0.06560689210891724,
+ 0.016219589859247208
+ ],
+ "p_fast": [
+ 0.02034814842045307,
+ 0.8979364633560181,
+ 0.06549247354269028,
+ 0.016222959384322166
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.12464269250631332,
+ 0.017863623797893524,
+ 0.28290945291519165,
+ 0.03845534846186638,
+ 0.016643228009343147,
+ 0.519485592842102
+ ],
+ "p_stock_fp16": [
+ 0.12470840662717819,
+ 0.01782613806426525,
+ 0.28213465213775635,
+ 0.038332220166921616,
+ 0.016551008448004723,
+ 0.5204476118087769
+ ],
+ "p_fast": [
+ 0.12538127601146698,
+ 0.01788734830915928,
+ 0.2825510501861572,
+ 0.03838879242539406,
+ 0.016607841476798058,
+ 0.5191836357116699
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999998807907104,
+ 1.394130180187858e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999998807907104,
+ 1.3887758143482642e-07
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 1.390472306184165e-07
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999376118183136,
+ 0.0006238745409063995
+ ],
+ "p_stock_fp16": [
+ 0.9993775486946106,
+ 0.0006225253455340862
+ ],
+ "p_fast": [
+ 0.9993730187416077,
+ 0.0006269456935115159
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.15760177373886108,
+ 0.4235360324382782,
+ 0.19814804196357727,
+ 0.22071416676044464
+ ],
+ "p_stock_fp16": [
+ 0.15763890743255615,
+ 0.42382505536079407,
+ 0.19801335036754608,
+ 0.22052264213562012
+ ],
+ "p_fast": [
+ 0.15729786455631256,
+ 0.4240453839302063,
+ 0.19801956415176392,
+ 0.22063729166984558
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.6595156788825989,
+ 0.01210157759487629,
+ 0.12298237532377243,
+ 0.037257686257362366,
+ 0.011656411923468113,
+ 0.1564863622188568
+ ],
+ "p_stock_fp16": [
+ 0.6596419215202332,
+ 0.012102426961064339,
+ 0.12282847613096237,
+ 0.037278104573488235,
+ 0.011661546304821968,
+ 0.15648755431175232
+ ],
+ "p_fast": [
+ 0.65907222032547,
+ 0.012062491849064827,
+ 0.12278233468532562,
+ 0.037373434752225876,
+ 0.011668555438518524,
+ 0.1570410132408142
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.22043702006340027,
+ 0.7795629501342773
+ ],
+ "p_stock_fp16": [
+ 0.21996520459651947,
+ 0.7800347805023193
+ ],
+ "p_fast": [
+ 0.21841919422149658,
+ 0.7815808057785034
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999945163726807,
+ 5.464833520818502e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999943971633911,
+ 5.6409676290059e-06
+ ],
+ "p_fast": [
+ 0.9999945163726807,
+ 5.51298717255122e-06
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.024277353659272194,
+ 0.3815690577030182,
+ 0.2951502501964569,
+ 0.29900333285331726
+ ],
+ "p_stock_fp16": [
+ 0.024232292547822,
+ 0.3812844157218933,
+ 0.29520973563194275,
+ 0.2992735505104065
+ ],
+ "p_fast": [
+ 0.02426256611943245,
+ 0.3813880681991577,
+ 0.2955785393714905,
+ 0.29877084493637085
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.8123772740364075,
+ 0.007107547949999571,
+ 0.11400726437568665,
+ 0.01736924983561039,
+ 0.004562469199299812,
+ 0.04457608237862587
+ ],
+ "p_stock_fp16": [
+ 0.8121925592422485,
+ 0.007102733477950096,
+ 0.1141853854060173,
+ 0.017340725287795067,
+ 0.004550181329250336,
+ 0.04462839290499687
+ ],
+ "p_fast": [
+ 0.8122994303703308,
+ 0.007096733897924423,
+ 0.11408895254135132,
+ 0.01735994964838028,
+ 0.004564132075756788,
+ 0.044590696692466736
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.999987006187439,
+ 1.3040490557614248e-05
+ ],
+ "p_stock_fp16": [
+ 0.9999877214431763,
+ 1.2222059012856334e-05
+ ],
+ "p_fast": [
+ 0.9999868869781494,
+ 1.3120349649398122e-05
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 4.2940371258737287e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 4.2536619559996325e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 4.258858155026246e-08
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.15674610435962677,
+ 0.35670819878578186,
+ 0.24389010667800903,
+ 0.24265554547309875
+ ],
+ "p_stock_fp16": [
+ 0.1566273421049118,
+ 0.35712578892707825,
+ 0.24365760385990143,
+ 0.24258917570114136
+ ],
+ "p_fast": [
+ 0.1566370129585266,
+ 0.3573223054409027,
+ 0.24379166960716248,
+ 0.2422490417957306
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.1868942379951477,
+ 0.05708451569080353,
+ 0.1850120574235916,
+ 0.43090909719467163,
+ 0.03413502871990204,
+ 0.10596512258052826
+ ],
+ "p_stock_fp16": [
+ 0.1864057183265686,
+ 0.057017333805561066,
+ 0.18414409458637238,
+ 0.43234947323799133,
+ 0.03407986834645271,
+ 0.106003537774086
+ ],
+ "p_fast": [
+ 0.18654420971870422,
+ 0.05686500668525696,
+ 0.18464119732379913,
+ 0.4320373833179474,
+ 0.03408854082226753,
+ 0.10582362860441208
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8947815895080566,
+ 0.10521838814020157
+ ],
+ "p_stock_fp16": [
+ 0.8940516710281372,
+ 0.10594828426837921
+ ],
+ "p_fast": [
+ 0.8946975469589233,
+ 0.10530251264572144
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8013536930084229,
+ 0.19864635169506073
+ ],
+ "p_stock_fp16": [
+ 0.8022845387458801,
+ 0.1977154016494751
+ ],
+ "p_fast": [
+ 0.8057851195335388,
+ 0.19421494007110596
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.03512261435389519,
+ 0.4689916670322418,
+ 0.3044058680534363,
+ 0.19147989153862
+ ],
+ "p_stock_fp16": [
+ 0.03528532385826111,
+ 0.4690678119659424,
+ 0.3041496276855469,
+ 0.19149722158908844
+ ],
+ "p_fast": [
+ 0.03504953533411026,
+ 0.4690720736980438,
+ 0.30444952845573425,
+ 0.19142882525920868
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.3087947368621826,
+ 0.043324682861566544,
+ 0.17265833914279938,
+ 0.10419898480176926,
+ 0.04842075705528259,
+ 0.32260245084762573
+ ],
+ "p_stock_fp16": [
+ 0.30881425738334656,
+ 0.04328890144824982,
+ 0.17255376279354095,
+ 0.10404770076274872,
+ 0.048292260617017746,
+ 0.3230031132698059
+ ],
+ "p_fast": [
+ 0.30839449167251587,
+ 0.043230053037405014,
+ 0.1726560741662979,
+ 0.10410939902067184,
+ 0.04841536656022072,
+ 0.3231946527957916
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7984353303909302,
+ 0.2015647143125534
+ ],
+ "p_stock_fp16": [
+ 0.7956580519676208,
+ 0.20434190332889557
+ ],
+ "p_fast": [
+ 0.7978129386901855,
+ 0.20218709111213684
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9876341819763184,
+ 0.012365822680294514
+ ],
+ "p_stock_fp16": [
+ 0.9876490235328674,
+ 0.012350981123745441
+ ],
+ "p_fast": [
+ 0.9874932765960693,
+ 0.012506808154284954
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.13348746299743652,
+ 0.6239736676216125,
+ 0.11300080269575119,
+ 0.12953810393810272
+ ],
+ "p_stock_fp16": [
+ 0.13412055373191833,
+ 0.6232026815414429,
+ 0.1130925789475441,
+ 0.1295841485261917
+ ],
+ "p_fast": [
+ 0.1332918256521225,
+ 0.6244192123413086,
+ 0.11285779625177383,
+ 0.1294310986995697
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.09394488483667374,
+ 0.015777451917529106,
+ 0.5153059959411621,
+ 0.04843524843454361,
+ 0.010970758274197578,
+ 0.3155655562877655
+ ],
+ "p_stock_fp16": [
+ 0.09421014785766602,
+ 0.015836616978049278,
+ 0.5150470733642578,
+ 0.0485900342464447,
+ 0.011012637987732887,
+ 0.3153035342693329
+ ],
+ "p_fast": [
+ 0.09393259137868881,
+ 0.015789959579706192,
+ 0.5145336389541626,
+ 0.0485415980219841,
+ 0.010980193503201008,
+ 0.31622201204299927
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999998807907104,
+ 1.318972095987192e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999998807907104,
+ 1.324693954529721e-07
+ ],
+ "p_fast": [
+ 0.9999998807907104,
+ 1.3289050571074768e-07
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.01026938296854496,
+ 0.9897305965423584
+ ],
+ "p_stock_fp16": [
+ 0.010258511640131474,
+ 0.9897414445877075
+ ],
+ "p_fast": [
+ 0.010248602367937565,
+ 0.9897513389587402
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.017897268757224083,
+ 0.4805293679237366,
+ 0.3867000639438629,
+ 0.11487327516078949
+ ],
+ "p_stock_fp16": [
+ 0.017916115000844002,
+ 0.4804691970348358,
+ 0.3868226408958435,
+ 0.11479207873344421
+ ],
+ "p_fast": [
+ 0.017893994227051735,
+ 0.4801103472709656,
+ 0.3872893750667572,
+ 0.11470634490251541
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.38514015078544617,
+ 0.015435699373483658,
+ 0.4762321412563324,
+ 0.0579715371131897,
+ 0.009694179520010948,
+ 0.05552630499005318
+ ],
+ "p_stock_fp16": [
+ 0.38524994254112244,
+ 0.015476035885512829,
+ 0.47601065039634705,
+ 0.05806472525000572,
+ 0.009684656746685505,
+ 0.05551406741142273
+ ],
+ "p_fast": [
+ 0.38353589177131653,
+ 0.015437299385666847,
+ 0.4776095449924469,
+ 0.05814608559012413,
+ 0.00967930257320404,
+ 0.055591851472854614
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9966810345649719,
+ 0.0033189146779477596
+ ],
+ "p_stock_fp16": [
+ 0.9966512322425842,
+ 0.003348793601617217
+ ],
+ "p_fast": [
+ 0.9966771602630615,
+ 0.003322819247841835
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999978542327881,
+ 2.1836560790688964e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999978542327881,
+ 2.2039248506189324e-06
+ ],
+ "p_fast": [
+ 0.9999978542327881,
+ 2.1982825728628086e-06
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.011736857704818249,
+ 0.3596506416797638,
+ 0.36514797806739807,
+ 0.2634645402431488
+ ],
+ "p_stock_fp16": [
+ 0.011764687485992908,
+ 0.3599628508090973,
+ 0.3649179935455322,
+ 0.26335442066192627
+ ],
+ "p_fast": [
+ 0.011773757636547089,
+ 0.3602403700351715,
+ 0.36519935727119446,
+ 0.2627864480018616
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.09688996523618698,
+ 0.00994786061346531,
+ 0.7770570516586304,
+ 0.02218216098845005,
+ 0.007498866878449917,
+ 0.08642419427633286
+ ],
+ "p_stock_fp16": [
+ 0.09753671288490295,
+ 0.00996400136500597,
+ 0.7762764096260071,
+ 0.022149324417114258,
+ 0.00749190105125308,
+ 0.08658166974782944
+ ],
+ "p_fast": [
+ 0.09739021956920624,
+ 0.009978228248655796,
+ 0.7761994004249573,
+ 0.022224312648177147,
+ 0.007502596825361252,
+ 0.0867052897810936
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9835925102233887,
+ 0.016407465562224388
+ ],
+ "p_stock_fp16": [
+ 0.98366379737854,
+ 0.01633620448410511
+ ],
+ "p_fast": [
+ 0.9836456775665283,
+ 0.016354359686374664
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9583739638328552,
+ 0.04162605106830597
+ ],
+ "p_stock_fp16": [
+ 0.9588655829429626,
+ 0.041134417057037354
+ ],
+ "p_fast": [
+ 0.9584791660308838,
+ 0.0415208600461483
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9999871253967285,
+ 1.0821215710166143e-06,
+ 4.877525043411879e-06,
+ 2.3712685504051478e-07,
+ 2.183285005230573e-06,
+ 4.476551566767739e-06
+ ],
+ "p_stock_fp16": [
+ 0.9999867677688599,
+ 1.1058933750973665e-06,
+ 5.009798769606277e-06,
+ 2.4198544679165934e-07,
+ 2.2427095700550126e-06,
+ 4.610738415067317e-06
+ ],
+ "p_fast": [
+ 0.9999867677688599,
+ 1.1058933750973665e-06,
+ 4.980530320608523e-06,
+ 2.4198544679165934e-07,
+ 2.2252565941016655e-06,
+ 4.610738415067317e-06
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9986361861228943,
+ 0.001363751944154501
+ ],
+ "p_stock_fp16": [
+ 0.9986335635185242,
+ 0.001366359181702137
+ ],
+ "p_fast": [
+ 0.998649537563324,
+ 0.00135046208743006
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9989473223686218,
+ 0.0010526657570153475
+ ],
+ "p_stock_fp16": [
+ 0.9989463686943054,
+ 0.0010535957990214229
+ ],
+ "p_fast": [
+ 0.9989423155784607,
+ 0.001057715155184269
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.0038620494306087494,
+ 0.15592151880264282,
+ 0.8402163982391357
+ ],
+ "p_stock_fp16": [
+ 0.0038611518684774637,
+ 0.15582279860973358,
+ 0.8403160572052002
+ ],
+ "p_fast": [
+ 0.0038683770690113306,
+ 0.15588587522506714,
+ 0.84024578332901
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9734492301940918,
+ 0.026550834998488426
+ ],
+ "p_stock_fp16": [
+ 0.973483681678772,
+ 0.026516366750001907
+ ],
+ "p_fast": [
+ 0.9735772013664246,
+ 0.026422783732414246
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.00241072871722281,
+ 0.9503281116485596,
+ 0.012125160545110703,
+ 0.003962733782827854,
+ 0.014108347706496716,
+ 0.017064949497580528
+ ],
+ "p_stock_fp16": [
+ 0.002407172927632928,
+ 0.9504814147949219,
+ 0.01208221074193716,
+ 0.0039532845839858055,
+ 0.014070466160774231,
+ 0.017005402594804764
+ ],
+ "p_fast": [
+ 0.0024070588406175375,
+ 0.9504364132881165,
+ 0.012128924019634724,
+ 0.003953097388148308,
+ 0.014069799333810806,
+ 0.017004597932100296
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 3.5781187790462354e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 3.657051905747721e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 3.657051905747721e-08
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9996920824050903,
+ 0.0003079473681282252
+ ],
+ "p_stock_fp16": [
+ 0.9996935129165649,
+ 0.0003065436612814665
+ ],
+ "p_fast": [
+ 0.9996856451034546,
+ 0.00031442419276572764
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.00017585146997589618,
+ 0.04481048509478569,
+ 0.9550136923789978
+ ],
+ "p_stock_fp16": [
+ 0.00017555132217239588,
+ 0.044798221439123154,
+ 0.9550261497497559
+ ],
+ "p_fast": [
+ 0.00017554365331307054,
+ 0.04484003037214279,
+ 0.9549844264984131
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.940037190914154,
+ 0.059962816536426544
+ ],
+ "p_stock_fp16": [
+ 0.9401335716247559,
+ 0.059866417199373245
+ ],
+ "p_fast": [
+ 0.9400647878646851,
+ 0.059935152530670166
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.1735127717256546,
+ 0.0018297028727829456,
+ 0.8000752329826355,
+ 0.0019193694461137056,
+ 0.003209521761164069,
+ 0.019453484565019608
+ ],
+ "p_stock_fp16": [
+ 0.17145508527755737,
+ 0.0018210249254480004,
+ 0.8021455407142639,
+ 0.0019158869981765747,
+ 0.003217920660972595,
+ 0.019444486126303673
+ ],
+ "p_fast": [
+ 0.17174029350280762,
+ 0.00182762008626014,
+ 0.8019120693206787,
+ 0.0019115919712930918,
+ 0.0032169839832931757,
+ 0.01939142495393753
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5092540979385376,
+ 0.49074587225914
+ ],
+ "p_stock_fp16": [
+ 0.5048826336860657,
+ 0.49511733651161194
+ ],
+ "p_fast": [
+ 0.5070796012878418,
+ 0.4929203987121582
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5292202830314636,
+ 0.47077974677085876
+ ],
+ "p_stock_fp16": [
+ 0.5253688097000122,
+ 0.4746311604976654
+ ],
+ "p_fast": [
+ 0.526829719543457,
+ 0.47317031025886536
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.1657688021659851,
+ 0.40832340717315674,
+ 0.42590779066085815
+ ],
+ "p_stock_fp16": [
+ 0.16552451252937317,
+ 0.40847864747047424,
+ 0.4259968101978302
+ ],
+ "p_fast": [
+ 0.16606754064559937,
+ 0.40861985087394714,
+ 0.4253125488758087
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9520440697669983,
+ 0.047955870628356934
+ ],
+ "p_stock_fp16": [
+ 0.9519749283790588,
+ 0.04802503064274788
+ ],
+ "p_fast": [
+ 0.952153205871582,
+ 0.047846756875514984
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.0002589143405202776,
+ 0.9902894496917725,
+ 0.00035433872835710645,
+ 0.0008422959363088012,
+ 0.0012613467406481504,
+ 0.0069936118088662624
+ ],
+ "p_stock_fp16": [
+ 0.0002582231827545911,
+ 0.9903140664100647,
+ 0.00035294925328344107,
+ 0.0008368171402253211,
+ 0.001258667092770338,
+ 0.006979268044233322
+ ],
+ "p_fast": [
+ 0.00025872947298921645,
+ 0.9903196096420288,
+ 0.00035364122595638037,
+ 0.0008368217968381941,
+ 0.0012586740776896477,
+ 0.0069724940694868565
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6833670735359192,
+ 0.3166328966617584
+ ],
+ "p_stock_fp16": [
+ 0.682600200176239,
+ 0.3173998296260834
+ ],
+ "p_fast": [
+ 0.6839210987091064,
+ 0.31607893109321594
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.04493996500968933,
+ 0.9550600051879883
+ ],
+ "p_stock_fp16": [
+ 0.045157428830862045,
+ 0.9548425674438477
+ ],
+ "p_fast": [
+ 0.045204825699329376,
+ 0.9547951817512512
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.0022316568065434694,
+ 0.08298008143901825,
+ 0.9147882461547852
+ ],
+ "p_stock_fp16": [
+ 0.002225915901362896,
+ 0.08295784145593643,
+ 0.9148162603378296
+ ],
+ "p_fast": [
+ 0.0022302581928670406,
+ 0.08295748382806778,
+ 0.9148122668266296
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9950529932975769,
+ 0.004947021137923002
+ ],
+ "p_stock_fp16": [
+ 0.9950292706489563,
+ 0.004970709327608347
+ ],
+ "p_fast": [
+ 0.9950733780860901,
+ 0.004926604684442282
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.014820062555372715,
+ 0.1400444358587265,
+ 0.03010142594575882,
+ 0.018572568893432617,
+ 0.06986543536186218,
+ 0.7265961170196533
+ ],
+ "p_stock_fp16": [
+ 0.014837374910712242,
+ 0.14049839973449707,
+ 0.030148452147841454,
+ 0.018610235303640366,
+ 0.07009725272655487,
+ 0.725808322429657
+ ],
+ "p_fast": [
+ 0.014800592325627804,
+ 0.14015011489391327,
+ 0.03007371537387371,
+ 0.018564101308584213,
+ 0.06992348283529282,
+ 0.726487934589386
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.1292426735162735,
+ 0.8707573413848877
+ ],
+ "p_stock_fp16": [
+ 0.1301264464855194,
+ 0.8698735237121582
+ ],
+ "p_fast": [
+ 0.12935803830623627,
+ 0.8706419467926025
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8551583886146545,
+ 0.14484164118766785
+ ],
+ "p_stock_fp16": [
+ 0.8542170524597168,
+ 0.1457829475402832
+ ],
+ "p_fast": [
+ 0.8548240661621094,
+ 0.14517594873905182
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.011988715268671513,
+ 0.12465611845254898,
+ 0.8633551597595215
+ ],
+ "p_stock_fp16": [
+ 0.011981947347521782,
+ 0.12473290413618088,
+ 0.8632851243019104
+ ],
+ "p_fast": [
+ 0.011966804973781109,
+ 0.12457526475191116,
+ 0.863457977771759
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9936420321464539,
+ 0.006358001381158829
+ ],
+ "p_stock_fp16": [
+ 0.9936413168907166,
+ 0.0063587515614926815
+ ],
+ "p_fast": [
+ 0.9936414361000061,
+ 0.006358558312058449
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.015540455467998981,
+ 0.04525015875697136,
+ 0.8563390374183655,
+ 0.05747560039162636,
+ 0.004162750672549009,
+ 0.021232010796666145
+ ],
+ "p_stock_fp16": [
+ 0.015654245391488075,
+ 0.04545215144753456,
+ 0.855527937412262,
+ 0.057805135846138,
+ 0.0041845799423754215,
+ 0.021375931799411774
+ ],
+ "p_fast": [
+ 0.015684867277741432,
+ 0.045430004596710205,
+ 0.8555288314819336,
+ 0.05778755620121956,
+ 0.004192764405161142,
+ 0.021375954151153564
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 3.0159040761645883e-05,
+ 0.9999698400497437
+ ],
+ "p_stock_fp16": [
+ 2.9773113055853173e-05,
+ 0.9999701976776123
+ ],
+ "p_fast": [
+ 2.807876626320649e-05,
+ 0.999971866607666
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 2.961303348803651e-13,
+ 1.0
+ ],
+ "p_stock_fp16": [
+ 2.9507036462643466e-13,
+ 1.0
+ ],
+ "p_fast": [
+ 2.973846212686593e-13,
+ 1.0
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.06593205034732819,
+ 0.19171085953712463,
+ 0.742357075214386
+ ],
+ "p_stock_fp16": [
+ 0.06594376266002655,
+ 0.19179539382457733,
+ 0.7422608137130737
+ ],
+ "p_fast": [
+ 0.06594376266002655,
+ 0.19179539382457733,
+ 0.7422608137130737
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.4913223385810852,
+ 0.5086777210235596
+ ],
+ "p_stock_fp16": [
+ 0.49047964811325073,
+ 0.5095202922821045
+ ],
+ "p_fast": [
+ 0.49047964811325073,
+ 0.5095202922821045
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.001022161915898323,
+ 0.9733814597129822,
+ 0.0017763517098501325,
+ 0.004195349756628275,
+ 0.0029587047174572945,
+ 0.016665969043970108
+ ],
+ "p_stock_fp16": [
+ 0.0010196493240073323,
+ 0.9733920693397522,
+ 0.0017721495823934674,
+ 0.004201638046652079,
+ 0.0029562211129814386,
+ 0.016658421605825424
+ ],
+ "p_fast": [
+ 0.0010236278176307678,
+ 0.9733802080154419,
+ 0.0017721279291436076,
+ 0.004201586823910475,
+ 0.002956185257062316,
+ 0.016666354611516
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 2.1680211403918292e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 2.169377921745763e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 2.168054358264726e-08
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9400132894515991,
+ 0.05998670682311058
+ ],
+ "p_stock_fp16": [
+ 0.9398581385612488,
+ 0.06014182046055794
+ ],
+ "p_fast": [
+ 0.9399133324623108,
+ 0.060086652636528015
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.00042098897392861545,
+ 0.0652046650648117,
+ 0.9343743324279785
+ ],
+ "p_stock_fp16": [
+ 0.00042096045217476785,
+ 0.06521910429000854,
+ 0.9343599677085876
+ ],
+ "p_fast": [
+ 0.00042096045217476785,
+ 0.06521910429000854,
+ 0.9343599677085876
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9980411529541016,
+ 0.0019587804563343525
+ ],
+ "p_stock_fp16": [
+ 0.9980377554893494,
+ 0.0019622696563601494
+ ],
+ "p_fast": [
+ 0.9980401396751404,
+ 0.0019598801154643297
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9930964112281799,
+ 0.0007562124519608915,
+ 0.005451478995382786,
+ 9.189159754896536e-05,
+ 0.00012555805733427405,
+ 0.000478360743727535
+ ],
+ "p_stock_fp16": [
+ 0.9930832386016846,
+ 0.0007581152021884918,
+ 0.0054613142274320126,
+ 9.179035987472162e-05,
+ 0.0001259535929420963,
+ 0.00047954011824913323
+ ],
+ "p_fast": [
+ 0.9930506944656372,
+ 0.0007610574248246849,
+ 0.005487866699695587,
+ 9.214659075951204e-05,
+ 0.00012594947475008667,
+ 0.00048234238056465983
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 4.0716045646149723e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 4.060359515278833e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 4.0578818527592375e-08
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9971466660499573,
+ 0.0028533684089779854
+ ],
+ "p_stock_fp16": [
+ 0.9971758127212524,
+ 0.0028242075350135565
+ ],
+ "p_fast": [
+ 0.9970850348472595,
+ 0.0029150154441595078
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.005973821505904198,
+ 0.22240279614925385,
+ 0.7716233730316162
+ ],
+ "p_stock_fp16": [
+ 0.005983010400086641,
+ 0.22254620492458344,
+ 0.771470844745636
+ ],
+ "p_fast": [
+ 0.005968203768134117,
+ 0.22221234440803528,
+ 0.7718194723129272
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9957072734832764,
+ 0.004292740952223539
+ ],
+ "p_stock_fp16": [
+ 0.9956894516944885,
+ 0.004310494754463434
+ ],
+ "p_fast": [
+ 0.9956958293914795,
+ 0.0043042125180363655
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.010118181817233562,
+ 0.4592422842979431,
+ 0.0095077408477664,
+ 0.043998055160045624,
+ 0.053700730204582214,
+ 0.4234329164028168
+ ],
+ "p_stock_fp16": [
+ 0.010095734149217606,
+ 0.4584484398365021,
+ 0.009484064765274525,
+ 0.043853867799043655,
+ 0.05394117906689644,
+ 0.42417672276496887
+ ],
+ "p_fast": [
+ 0.010160012170672417,
+ 0.45850396156311035,
+ 0.009525826200842857,
+ 0.04387524351477623,
+ 0.05386217311024666,
+ 0.4240727424621582
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.0017355539603158832,
+ 0.998264491558075
+ ],
+ "p_stock_fp16": [
+ 0.0017358994809910655,
+ 0.9982641339302063
+ ],
+ "p_fast": [
+ 0.0017274586716666818,
+ 0.9982725381851196
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 7.675309461774305e-06,
+ 0.9999923706054688
+ ],
+ "p_stock_fp16": [
+ 7.58703163228347e-06,
+ 0.9999923706054688
+ ],
+ "p_fast": [
+ 7.889262633398175e-06,
+ 0.9999921321868896
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.00750641617923975,
+ 0.16059726476669312,
+ 0.8318963050842285
+ ],
+ "p_stock_fp16": [
+ 0.007483856286853552,
+ 0.16063816845417023,
+ 0.8318779468536377
+ ],
+ "p_fast": [
+ 0.007469948846846819,
+ 0.16057468950748444,
+ 0.8319553136825562
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.003097604028880596,
+ 0.9969024062156677
+ ],
+ "p_stock_fp16": [
+ 0.0030994152184575796,
+ 0.9969005584716797
+ ],
+ "p_fast": [
+ 0.0030843650456517935,
+ 0.9969155788421631
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.4055050313472748,
+ 0.15206968784332275,
+ 0.04695017263293266,
+ 0.007017936557531357,
+ 0.3164553940296173,
+ 0.07200168818235397
+ ],
+ "p_stock_fp16": [
+ 0.4055476486682892,
+ 0.1515790820121765,
+ 0.04650067910552025,
+ 0.006965918000787497,
+ 0.3180849254131317,
+ 0.07132174074649811
+ ],
+ "p_fast": [
+ 0.40653666853904724,
+ 0.15176337957382202,
+ 0.04678510129451752,
+ 0.006994850002229214,
+ 0.31630203127861023,
+ 0.0716179758310318
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8976332545280457,
+ 0.10236673802137375
+ ],
+ "p_stock_fp16": [
+ 0.8978521227836609,
+ 0.10214786231517792
+ ],
+ "p_fast": [
+ 0.8984551429748535,
+ 0.10154489427804947
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9451290369033813,
+ 0.05487101525068283
+ ],
+ "p_stock_fp16": [
+ 0.9448422193527222,
+ 0.05515776947140694
+ ],
+ "p_fast": [
+ 0.9446127414703369,
+ 0.05538724362850189
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.006031727883964777,
+ 0.10943283885717392,
+ 0.8845353722572327
+ ],
+ "p_stock_fp16": [
+ 0.0060361335054039955,
+ 0.10942338407039642,
+ 0.8845404982566833
+ ],
+ "p_fast": [
+ 0.006030920892953873,
+ 0.10932888835668564,
+ 0.8846402168273926
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999843835830688,
+ 1.5644949598936364e-05
+ ],
+ "p_stock_fp16": [
+ 0.9999842643737793,
+ 1.5783676644787192e-05
+ ],
+ "p_fast": [
+ 0.9999847412109375,
+ 1.5249594071065076e-05
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.0033057723194360733,
+ 0.6169611811637878,
+ 0.002336987294256687,
+ 0.005646237637847662,
+ 0.004208066035062075,
+ 0.36754176020622253
+ ],
+ "p_stock_fp16": [
+ 0.0033101094886660576,
+ 0.6161828637123108,
+ 0.0023380678612738848,
+ 0.0056527238339185715,
+ 0.004217188805341721,
+ 0.36829909682273865
+ ],
+ "p_fast": [
+ 0.0033166082575917244,
+ 0.6161879301071167,
+ 0.0023426583502441645,
+ 0.005641740281134844,
+ 0.004208994563668966,
+ 0.3683021366596222
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 3.5180260482547965e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 3.541508419857564e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 3.488305466703423e-08
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 3.765927658605506e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 3.823211969233853e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 3.822278671350432e-08
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.005241339560598135,
+ 0.2050454020500183,
+ 0.7897133231163025
+ ],
+ "p_stock_fp16": [
+ 0.005248962435871363,
+ 0.20501220226287842,
+ 0.7897388935089111
+ ],
+ "p_fast": [
+ 0.005201337393373251,
+ 0.20454558730125427,
+ 0.7902531027793884
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9699829816818237,
+ 0.03001696616411209
+ ],
+ "p_stock_fp16": [
+ 0.9700134992599487,
+ 0.029986508190631866
+ ],
+ "p_fast": [
+ 0.9701269268989563,
+ 0.029873093590140343
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.5469303131103516,
+ 0.0464814119040966,
+ 0.019957855343818665,
+ 0.005070303566753864,
+ 0.014565481804311275,
+ 0.366994708776474
+ ],
+ "p_stock_fp16": [
+ 0.5460989475250244,
+ 0.04671189561486244,
+ 0.020012209191918373,
+ 0.005089614074677229,
+ 0.014669870026409626,
+ 0.36741751432418823
+ ],
+ "p_fast": [
+ 0.5467650294303894,
+ 0.04675174877047539,
+ 0.020029284060001373,
+ 0.005084016360342503,
+ 0.014625145122408867,
+ 0.3667447865009308
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 1.4582973051346926e-08
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 1.4639378598246822e-08
+ ],
+ "p_fast": [
+ 1.0,
+ 1.4639378598246822e-08
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 1.0,
+ 7.966840520623464e-09
+ ],
+ "p_stock_fp16": [
+ 1.0,
+ 7.932133172516842e-09
+ ],
+ "p_fast": [
+ 1.0,
+ 7.916656663553567e-09
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.0016410114476457238,
+ 0.09309486299753189,
+ 0.9052640795707703
+ ],
+ "p_stock_fp16": [
+ 0.0016416601138189435,
+ 0.09311114996671677,
+ 0.9052472114562988
+ ],
+ "p_fast": [
+ 0.00164792500436306,
+ 0.09319304674863815,
+ 0.9051589965820312
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8592309951782227,
+ 0.14076896011829376
+ ],
+ "p_stock_fp16": [
+ 0.8589553833007812,
+ 0.14104463160037994
+ ],
+ "p_fast": [
+ 0.8600167632102966,
+ 0.1399831771850586
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/benchmarks/results/parity_typed_decisions_fp16_rtx4070.json b/benchmarks/results/parity_typed_decisions_fp16_rtx4070.json
new file mode 100644
index 00000000..ae598f8c
--- /dev/null
+++ b/benchmarks/results/parity_typed_decisions_fp16_rtx4070.json
@@ -0,0 +1,6122 @@
+{
+ "model": "convaiinnovations/laya-typed-decisions",
+ "subfolder": null,
+ "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER",
+ "torch": "2.11.0+cu130",
+ "dtype": "torch.float16",
+ "summary": {
+ "choice": {
+ "n": 48,
+ "d_fast_stock": 0.001519322395324707,
+ "d_fast_fp32": 0.0015984773635864258,
+ "d_stock_fp32": 0.0012166500091552734,
+ "agree_fast_stock": 48,
+ "agree_fast_fp32": 47,
+ "agree_stock_fp32": 47
+ },
+ "noul": {
+ "n": 180,
+ "d_fast_stock": 0.005261600017547607,
+ "d_fast_fp32": 0.005308419466018677,
+ "d_stock_fp32": 0.003399401903152466,
+ "agree_fast_stock": 180,
+ "agree_fast_fp32": 180,
+ "agree_stock_fp32": 180
+ },
+ "score": {
+ "n": 60,
+ "d_fast_stock": 0.0012440085411071777,
+ "d_fast_fp32": 0.0012798905372619629,
+ "d_stock_fp32": 0.0019672513008117676,
+ "agree_fast_stock": 60,
+ "agree_fast_fp32": 60,
+ "agree_stock_fp32": 60
+ }
+ },
+ "cases": [
+ {
+ "state": "triage/0",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.6927211284637451,
+ 0.014246546663343906,
+ 0.13677994906902313,
+ 0.007951270788908005,
+ 0.10968707501888275,
+ 0.038614001125097275
+ ],
+ "p_stock_fp16": [
+ 0.6926677227020264,
+ 0.014236154966056347,
+ 0.13686135411262512,
+ 0.007939103990793228,
+ 0.1097298189997673,
+ 0.03856584057211876
+ ],
+ "p_fast": [
+ 0.6931349635124207,
+ 0.014231852255761623,
+ 0.13655303418636322,
+ 0.00795999076217413,
+ 0.10950935631990433,
+ 0.03861070051789284
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.4151865839958191,
+ 0.5848134160041809
+ ],
+ "p_stock_fp16": [
+ 0.41560280323028564,
+ 0.5843971967697144
+ ],
+ "p_fast": [
+ 0.41561025381088257,
+ 0.5843898057937622
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.02600688301026821,
+ 0.15357312560081482,
+ 0.5514219403266907,
+ 0.26899799704551697
+ ],
+ "p_stock_fp16": [
+ 0.025973951444029808,
+ 0.1533883661031723,
+ 0.5515642166137695,
+ 0.2690735459327698
+ ],
+ "p_fast": [
+ 0.025991834700107574,
+ 0.15334415435791016,
+ 0.5514051914215088,
+ 0.2692588269710541
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3149517774581909,
+ 0.6850481629371643
+ ],
+ "p_stock_fp16": [
+ 0.3149124085903168,
+ 0.6850876212120056
+ ],
+ "p_fast": [
+ 0.31502431631088257,
+ 0.6849756836891174
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.16822032630443573,
+ 0.8317797183990479
+ ],
+ "p_stock_fp16": [
+ 0.16813862323760986,
+ 0.8318613767623901
+ ],
+ "p_fast": [
+ 0.16795091331005096,
+ 0.8320490717887878
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.006520497612655163,
+ 0.9102498888969421,
+ 0.008270232938230038,
+ 0.009572864510118961,
+ 0.00717336218804121,
+ 0.05821319296956062
+ ],
+ "p_stock_fp16": [
+ 0.006515983492136002,
+ 0.9102426767349243,
+ 0.008269212208688259,
+ 0.009555058553814888,
+ 0.007170398253947496,
+ 0.05824671685695648
+ ],
+ "p_fast": [
+ 0.006517169997096062,
+ 0.9104084372520447,
+ 0.00825458113104105,
+ 0.009575482457876205,
+ 0.007185724563896656,
+ 0.058058544993400574
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.20784282684326172,
+ 0.7921571731567383
+ ],
+ "p_stock_fp16": [
+ 0.207977756857872,
+ 0.792022168636322
+ ],
+ "p_fast": [
+ 0.20763616263866425,
+ 0.7923638820648193
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.035688433796167374,
+ 0.2066349983215332,
+ 0.46579089760780334,
+ 0.29188570380210876
+ ],
+ "p_stock_fp16": [
+ 0.03564111515879631,
+ 0.20691105723381042,
+ 0.46537140011787415,
+ 0.29207643866539
+ ],
+ "p_fast": [
+ 0.035710424184799194,
+ 0.2067069262266159,
+ 0.4653666019439697,
+ 0.29221609234809875
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9487811923027039,
+ 0.05121885612607002
+ ],
+ "p_stock_fp16": [
+ 0.9493533968925476,
+ 0.05064664036035538
+ ],
+ "p_fast": [
+ 0.9490472674369812,
+ 0.050952713936567307
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9140059947967529,
+ 0.08599407225847244
+ ],
+ "p_stock_fp16": [
+ 0.9136187434196472,
+ 0.08638126403093338
+ ],
+ "p_fast": [
+ 0.9139840602874756,
+ 0.08601587265729904
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.029065648093819618,
+ 0.027133196592330933,
+ 0.21221938729286194,
+ 0.3385237455368042,
+ 0.022455455735325813,
+ 0.370602548122406
+ ],
+ "p_stock_fp16": [
+ 0.029042411595582962,
+ 0.027070501819252968,
+ 0.2125624269247055,
+ 0.33818432688713074,
+ 0.02244223840534687,
+ 0.37069812417030334
+ ],
+ "p_fast": [
+ 0.029105432331562042,
+ 0.027129245921969414,
+ 0.21219317615032196,
+ 0.3382568955421448,
+ 0.022447049617767334,
+ 0.37086817622184753
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9442138075828552,
+ 0.0557861328125
+ ],
+ "p_stock_fp16": [
+ 0.9442282915115356,
+ 0.05577168986201286
+ ],
+ "p_fast": [
+ 0.9443310499191284,
+ 0.05566892400383949
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.6581481695175171,
+ 0.2644989490509033,
+ 0.05550163611769676,
+ 0.02185126207768917
+ ],
+ "p_stock_fp16": [
+ 0.6579216122627258,
+ 0.26465871930122375,
+ 0.05555681511759758,
+ 0.021862853318452835
+ ],
+ "p_fast": [
+ 0.6581591963768005,
+ 0.2644958794116974,
+ 0.05549551546573639,
+ 0.02184939943253994
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9795986413955688,
+ 0.020401325076818466
+ ],
+ "p_stock_fp16": [
+ 0.9796968102455139,
+ 0.02030319720506668
+ ],
+ "p_fast": [
+ 0.9796189069747925,
+ 0.02038104087114334
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9609938859939575,
+ 0.039006058126688004
+ ],
+ "p_stock_fp16": [
+ 0.9610797762870789,
+ 0.038920238614082336
+ ],
+ "p_fast": [
+ 0.9610066413879395,
+ 0.038993366062641144
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.011759733781218529,
+ 0.10597094893455505,
+ 0.027097292244434357,
+ 0.1957862675189972,
+ 0.09853699803352356,
+ 0.5608487725257874
+ ],
+ "p_stock_fp16": [
+ 0.01176037359982729,
+ 0.10608052462339401,
+ 0.02707795798778534,
+ 0.1957799792289734,
+ 0.0987090989947319,
+ 0.5605920553207397
+ ],
+ "p_fast": [
+ 0.011765589937567711,
+ 0.10625721514225006,
+ 0.02708996832370758,
+ 0.1961539387702942,
+ 0.09843995422124863,
+ 0.5602933168411255
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9739807844161987,
+ 0.026019226759672165
+ ],
+ "p_stock_fp16": [
+ 0.9740055203437805,
+ 0.025994420051574707
+ ],
+ "p_fast": [
+ 0.9740055203437805,
+ 0.025994420051574707
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.20646154880523682,
+ 0.22206242382526398,
+ 0.4460499584674835,
+ 0.1254260540008545
+ ],
+ "p_stock_fp16": [
+ 0.20642408728599548,
+ 0.22227323055267334,
+ 0.44605422019958496,
+ 0.12524840235710144
+ ],
+ "p_fast": [
+ 0.20616024732589722,
+ 0.22215178608894348,
+ 0.44635507464408875,
+ 0.12533287703990936
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999996423721313,
+ 3.830248829217453e-07
+ ],
+ "p_stock_fp16": [
+ 0.9999996423721313,
+ 3.7571533084701514e-07
+ ],
+ "p_fast": [
+ 0.9999996423721313,
+ 3.9936151097208494e-07
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9140607714653015,
+ 0.08593921363353729
+ ],
+ "p_stock_fp16": [
+ 0.9139937162399292,
+ 0.0860062763094902
+ ],
+ "p_fast": [
+ 0.9139937162399292,
+ 0.0860062763094902
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.051845304667949677,
+ 0.021178212016820908,
+ 0.01883244886994362,
+ 0.025343209505081177,
+ 0.04202444478869438,
+ 0.8407763242721558
+ ],
+ "p_stock_fp16": [
+ 0.052113067358732224,
+ 0.021220721304416656,
+ 0.01887410320341587,
+ 0.025348322466015816,
+ 0.04220244660973549,
+ 0.8402413129806519
+ ],
+ "p_fast": [
+ 0.05170731246471405,
+ 0.021137908101081848,
+ 0.018800443038344383,
+ 0.025298763066530228,
+ 0.04199671000242233,
+ 0.8410589098930359
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9925909042358398,
+ 0.007409045938402414
+ ],
+ "p_stock_fp16": [
+ 0.9926005005836487,
+ 0.007399426307529211
+ ],
+ "p_fast": [
+ 0.9925790429115295,
+ 0.007420975714921951
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.01001009251922369,
+ 0.023612383753061295,
+ 0.21541395783424377,
+ 0.750963568687439
+ ],
+ "p_stock_fp16": [
+ 0.0100228451192379,
+ 0.0236246045678854,
+ 0.21545200049877167,
+ 0.7509005069732666
+ ],
+ "p_fast": [
+ 0.010003483854234219,
+ 0.023625068366527557,
+ 0.21545621752738953,
+ 0.7509152293205261
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9891748428344727,
+ 0.010825135745108128
+ ],
+ "p_stock_fp16": [
+ 0.9891658425331116,
+ 0.010834136046469212
+ ],
+ "p_fast": [
+ 0.9891710877418518,
+ 0.010828904807567596
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9516403675079346,
+ 0.04835968092083931
+ ],
+ "p_stock_fp16": [
+ 0.9517288208007812,
+ 0.04827118292450905
+ ],
+ "p_fast": [
+ 0.9516221284866333,
+ 0.04837784543633461
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.03645458072423935,
+ 0.0668901652097702,
+ 0.12994635105133057,
+ 0.20574554800987244,
+ 0.08423464745283127,
+ 0.4767287075519562
+ ],
+ "p_stock_fp16": [
+ 0.036498915404081345,
+ 0.06673957407474518,
+ 0.12990634143352509,
+ 0.20569787919521332,
+ 0.08411992341279984,
+ 0.4770374298095703
+ ],
+ "p_fast": [
+ 0.03637709841132164,
+ 0.06677716970443726,
+ 0.13010650873184204,
+ 0.20596453547477722,
+ 0.08416730165481567,
+ 0.47660747170448303
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3825701177120209,
+ 0.6174299120903015
+ ],
+ "p_stock_fp16": [
+ 0.38249456882476807,
+ 0.6175054311752319
+ ],
+ "p_fast": [
+ 0.3822999596595764,
+ 0.6177000403404236
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.30471915006637573,
+ 0.2209567427635193,
+ 0.34811830520629883,
+ 0.12620577216148376
+ ],
+ "p_stock_fp16": [
+ 0.3043084740638733,
+ 0.22095820307731628,
+ 0.34846577048301697,
+ 0.12626756727695465
+ ],
+ "p_fast": [
+ 0.30462443828582764,
+ 0.22072909772396088,
+ 0.3484020233154297,
+ 0.1262444704771042
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.914557933807373,
+ 0.08544203639030457
+ ],
+ "p_stock_fp16": [
+ 0.9146726131439209,
+ 0.08532741665840149
+ ],
+ "p_fast": [
+ 0.9146726131439209,
+ 0.08532741665840149
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9264069199562073,
+ 0.07359310984611511
+ ],
+ "p_stock_fp16": [
+ 0.9264535307884216,
+ 0.0735464096069336
+ ],
+ "p_fast": [
+ 0.9261202216148376,
+ 0.07387980818748474
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.039129696786403656,
+ 0.19713838398456573,
+ 0.025492748245596886,
+ 0.034642286598682404,
+ 0.16664962470531464,
+ 0.5369472503662109
+ ],
+ "p_stock_fp16": [
+ 0.03907061740756035,
+ 0.19651871919631958,
+ 0.02549832873046398,
+ 0.034682322293519974,
+ 0.16671712696552277,
+ 0.5375128984451294
+ ],
+ "p_fast": [
+ 0.0390709824860096,
+ 0.197500541806221,
+ 0.02547367662191391,
+ 0.03468264266848564,
+ 0.16706499457359314,
+ 0.5362071990966797
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7484206557273865,
+ 0.25157931447029114
+ ],
+ "p_stock_fp16": [
+ 0.7463687062263489,
+ 0.2536312937736511
+ ],
+ "p_fast": [
+ 0.7471593022346497,
+ 0.25284066796302795
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.05583375692367554,
+ 0.09954751282930374,
+ 0.4533383846282959,
+ 0.3912803530693054
+ ],
+ "p_stock_fp16": [
+ 0.05559482425451279,
+ 0.09939926862716675,
+ 0.45359793305397034,
+ 0.39140787720680237
+ ],
+ "p_fast": [
+ 0.05609958618879318,
+ 0.09950903803110123,
+ 0.4532681405544281,
+ 0.39112329483032227
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6007707715034485,
+ 0.3992292284965515
+ ],
+ "p_stock_fp16": [
+ 0.5990967154502869,
+ 0.40090328454971313
+ ],
+ "p_fast": [
+ 0.6009936332702637,
+ 0.3990063965320587
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8552542924880981,
+ 0.1447456330060959
+ ],
+ "p_stock_fp16": [
+ 0.8547672629356384,
+ 0.14523276686668396
+ ],
+ "p_fast": [
+ 0.8554478287696838,
+ 0.14455218613147736
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.01829460635781288,
+ 0.04202770069241524,
+ 0.2798268795013428,
+ 0.2802577018737793,
+ 0.010803116485476494,
+ 0.36879006028175354
+ ],
+ "p_stock_fp16": [
+ 0.01827051118016243,
+ 0.04206738620996475,
+ 0.2797241508960724,
+ 0.28054487705230713,
+ 0.010803799144923687,
+ 0.36858925223350525
+ ],
+ "p_fast": [
+ 0.01830286532640457,
+ 0.042018596082925797,
+ 0.27939972281455994,
+ 0.2796727120876312,
+ 0.010822928510606289,
+ 0.3697832226753235
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9810092449188232,
+ 0.01899082027375698
+ ],
+ "p_stock_fp16": [
+ 0.9810583591461182,
+ 0.018941707909107208
+ ],
+ "p_fast": [
+ 0.9811036586761475,
+ 0.018896393477916718
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.4946100413799286,
+ 0.37923893332481384,
+ 0.10838776081800461,
+ 0.017763283103704453
+ ],
+ "p_stock_fp16": [
+ 0.4944571554660797,
+ 0.37911346554756165,
+ 0.10861119627952576,
+ 0.017818162217736244
+ ],
+ "p_fast": [
+ 0.4949454069137573,
+ 0.3787473440170288,
+ 0.10850630700588226,
+ 0.01780095510184765
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9998832941055298,
+ 0.00011671962420223281
+ ],
+ "p_stock_fp16": [
+ 0.9998750686645508,
+ 0.00012490990047808737
+ ],
+ "p_fast": [
+ 0.9998781681060791,
+ 0.00012183813669253141
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9885086417198181,
+ 0.011491389945149422
+ ],
+ "p_stock_fp16": [
+ 0.9886188507080078,
+ 0.011381092481315136
+ ],
+ "p_fast": [
+ 0.9885804057121277,
+ 0.011419614776968956
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.11449223011732101,
+ 0.2214956432580948,
+ 0.2832673192024231,
+ 0.022097542881965637,
+ 0.015251493081450462,
+ 0.3433957099914551
+ ],
+ "p_stock_fp16": [
+ 0.11437095701694489,
+ 0.22164304554462433,
+ 0.28307080268859863,
+ 0.0221339613199234,
+ 0.015242175199091434,
+ 0.3435389995574951
+ ],
+ "p_fast": [
+ 0.11439552903175354,
+ 0.22053006291389465,
+ 0.2836851477622986,
+ 0.02212250791490078,
+ 0.015234287828207016,
+ 0.34403249621391296
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5090459585189819,
+ 0.49095404148101807
+ ],
+ "p_stock_fp16": [
+ 0.5085440874099731,
+ 0.49145591259002686
+ ],
+ "p_fast": [
+ 0.5085745453834534,
+ 0.49142539501190186
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.015272198244929314,
+ 0.1495506316423416,
+ 0.5121871829032898,
+ 0.3229900002479553
+ ],
+ "p_stock_fp16": [
+ 0.01525065116584301,
+ 0.14936016499996185,
+ 0.512484610080719,
+ 0.3229045271873474
+ ],
+ "p_fast": [
+ 0.015276139602065086,
+ 0.14975595474243164,
+ 0.5118394494056702,
+ 0.32312849164009094
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9280251860618591,
+ 0.07197485864162445
+ ],
+ "p_stock_fp16": [
+ 0.9278388023376465,
+ 0.0721612349152565
+ ],
+ "p_fast": [
+ 0.9280672669410706,
+ 0.07193271070718765
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9008024334907532,
+ 0.0991976410150528
+ ],
+ "p_stock_fp16": [
+ 0.900933563709259,
+ 0.099066361784935
+ ],
+ "p_fast": [
+ 0.9007209539413452,
+ 0.099279023706913
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.06780702620744705,
+ 0.08060313016176224,
+ 0.16539226472377777,
+ 0.18834419548511505,
+ 0.028704779222607613,
+ 0.4691486060619354
+ ],
+ "p_stock_fp16": [
+ 0.06792692095041275,
+ 0.08058647811412811,
+ 0.16555467247962952,
+ 0.18821734189987183,
+ 0.02873399294912815,
+ 0.4689805507659912
+ ],
+ "p_fast": [
+ 0.06769228726625443,
+ 0.08062243461608887,
+ 0.16570945084095,
+ 0.18832431733608246,
+ 0.028690721839666367,
+ 0.4689607620239258
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.933054506778717,
+ 0.06694552302360535
+ ],
+ "p_stock_fp16": [
+ 0.9331572651863098,
+ 0.06684277951717377
+ ],
+ "p_fast": [
+ 0.9325762391090393,
+ 0.06742378324270248
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.336121529340744,
+ 0.26378756761550903,
+ 0.28905773162841797,
+ 0.11103314161300659
+ ],
+ "p_stock_fp16": [
+ 0.33657363057136536,
+ 0.26366421580314636,
+ 0.28873133659362793,
+ 0.11103086173534393
+ ],
+ "p_fast": [
+ 0.3358028531074524,
+ 0.26357465982437134,
+ 0.28948014974594116,
+ 0.11114230751991272
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9875450730323792,
+ 0.012454889714717865
+ ],
+ "p_stock_fp16": [
+ 0.9878117442131042,
+ 0.012188252992928028
+ ],
+ "p_fast": [
+ 0.9875383377075195,
+ 0.01246166042983532
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9731279611587524,
+ 0.02687203325331211
+ ],
+ "p_stock_fp16": [
+ 0.9731132388114929,
+ 0.026886744424700737
+ ],
+ "p_fast": [
+ 0.973087728023529,
+ 0.026912305504083633
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.007776996120810509,
+ 0.8459128141403198,
+ 0.009180194698274136,
+ 0.018750963732600212,
+ 0.009265405125916004,
+ 0.10911355167627335
+ ],
+ "p_stock_fp16": [
+ 0.007788767106831074,
+ 0.8457151651382446,
+ 0.00917741097509861,
+ 0.01875740848481655,
+ 0.009267473593354225,
+ 0.10929382592439651
+ ],
+ "p_fast": [
+ 0.007794814649969339,
+ 0.8455457091331482,
+ 0.009202493354678154,
+ 0.018790313974022865,
+ 0.009274668991565704,
+ 0.10939205437898636
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.838813066482544,
+ 0.16118696331977844
+ ],
+ "p_stock_fp16": [
+ 0.8390262722969055,
+ 0.1609737128019333
+ ],
+ "p_fast": [
+ 0.8390427827835083,
+ 0.1609572172164917
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.07716948539018631,
+ 0.32050901651382446,
+ 0.4018648862838745,
+ 0.20045654475688934
+ ],
+ "p_stock_fp16": [
+ 0.07735131680965424,
+ 0.32100388407707214,
+ 0.4014512002468109,
+ 0.20019365847110748
+ ],
+ "p_fast": [
+ 0.07740427553653717,
+ 0.320440411567688,
+ 0.401922345161438,
+ 0.20023299753665924
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9491718411445618,
+ 0.05082815885543823
+ ],
+ "p_stock_fp16": [
+ 0.9487513303756714,
+ 0.05124867707490921
+ ],
+ "p_fast": [
+ 0.9486562609672546,
+ 0.05134372040629387
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9675112366676331,
+ 0.032488688826560974
+ ],
+ "p_stock_fp16": [
+ 0.9675024151802063,
+ 0.032497506588697433
+ ],
+ "p_fast": [
+ 0.967563807964325,
+ 0.032436154782772064
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.02766232006251812,
+ 0.07833411544561386,
+ 0.4743930995464325,
+ 0.03598523139953613,
+ 0.014720987528562546,
+ 0.3689042031764984
+ ],
+ "p_stock_fp16": [
+ 0.027634689584374428,
+ 0.07849361002445221,
+ 0.47453415393829346,
+ 0.03593699261546135,
+ 0.0147341163828969,
+ 0.368666410446167
+ ],
+ "p_fast": [
+ 0.02770942822098732,
+ 0.07824607193470001,
+ 0.4739624857902527,
+ 0.03603418543934822,
+ 0.014745136722922325,
+ 0.3693026602268219
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9626188278198242,
+ 0.03738115355372429
+ ],
+ "p_stock_fp16": [
+ 0.9626556038856506,
+ 0.03734444081783295
+ ],
+ "p_fast": [
+ 0.9625676870346069,
+ 0.03743230178952217
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5732780694961548,
+ 0.26211413741111755,
+ 0.1290503442287445,
+ 0.03555748984217644
+ ],
+ "p_stock_fp16": [
+ 0.5726792812347412,
+ 0.26231974363327026,
+ 0.12934352457523346,
+ 0.03565746918320656
+ ],
+ "p_fast": [
+ 0.5729434490203857,
+ 0.2623126208782196,
+ 0.1291743367910385,
+ 0.035569556057453156
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.996161699295044,
+ 0.0038383081555366516
+ ],
+ "p_stock_fp16": [
+ 0.9961811304092407,
+ 0.003818906843662262
+ ],
+ "p_fast": [
+ 0.9961811304092407,
+ 0.003818906843662262
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9891064763069153,
+ 0.01089353859424591
+ ],
+ "p_stock_fp16": [
+ 0.9891344308853149,
+ 0.010865578427910805
+ ],
+ "p_fast": [
+ 0.9891658425331116,
+ 0.010834136046469212
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9249034523963928,
+ 0.0750965103507042
+ ],
+ "p_stock_fp16": [
+ 0.9249932169914246,
+ 0.07500684261322021
+ ],
+ "p_fast": [
+ 0.9250440001487732,
+ 0.07495604455471039
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9139643311500549,
+ 0.0860356017947197
+ ],
+ "p_stock_fp16": [
+ 0.9138977527618408,
+ 0.08610229194164276
+ ],
+ "p_fast": [
+ 0.9138593077659607,
+ 0.0861407220363617
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6928279399871826,
+ 0.307172030210495
+ ],
+ "p_stock_fp16": [
+ 0.6926484704017639,
+ 0.3073515295982361
+ ],
+ "p_fast": [
+ 0.6930511593818665,
+ 0.3069489002227783
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8171003460884094,
+ 0.1828996241092682
+ ],
+ "p_stock_fp16": [
+ 0.8177633285522461,
+ 0.1822367012500763
+ ],
+ "p_fast": [
+ 0.8173194527626038,
+ 0.18268054723739624
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.12013990432024002,
+ 0.15027251839637756,
+ 0.3511520326137543,
+ 0.37843558192253113
+ ],
+ "p_stock_fp16": [
+ 0.11989311873912811,
+ 0.15030166506767273,
+ 0.3510116636753082,
+ 0.37879353761672974
+ ],
+ "p_fast": [
+ 0.12018143385648727,
+ 0.1503094881772995,
+ 0.35086923837661743,
+ 0.3786398768424988
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.955962061882019,
+ 0.04403799772262573
+ ],
+ "p_stock_fp16": [
+ 0.9560223817825317,
+ 0.04397757351398468
+ ],
+ "p_fast": [
+ 0.9560223817825317,
+ 0.04397757351398468
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9811826348304749,
+ 0.01881740801036358
+ ],
+ "p_stock_fp16": [
+ 0.9811939597129822,
+ 0.018806083127856255
+ ],
+ "p_fast": [
+ 0.9811939597129822,
+ 0.018806083127856255
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8559523820877075,
+ 0.14404763281345367
+ ],
+ "p_stock_fp16": [
+ 0.8556552529335022,
+ 0.1443447470664978
+ ],
+ "p_fast": [
+ 0.8561821579933167,
+ 0.14381784200668335
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8014840483665466,
+ 0.19851592183113098
+ ],
+ "p_stock_fp16": [
+ 0.8022940158843994,
+ 0.19770601391792297
+ ],
+ "p_fast": [
+ 0.8017042875289917,
+ 0.1982957273721695
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2570229768753052,
+ 0.1490139216184616,
+ 0.3076266944408417,
+ 0.28633642196655273
+ ],
+ "p_stock_fp16": [
+ 0.25669190287590027,
+ 0.14885231852531433,
+ 0.30774548649787903,
+ 0.28671029210090637
+ ],
+ "p_fast": [
+ 0.25649914145469666,
+ 0.14859536290168762,
+ 0.3072892129421234,
+ 0.2876163125038147
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9851654767990112,
+ 0.014834562316536903
+ ],
+ "p_stock_fp16": [
+ 0.9851861596107483,
+ 0.014813886024057865
+ ],
+ "p_fast": [
+ 0.9851861596107483,
+ 0.014813886024057865
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9851340055465698,
+ 0.014866044744849205
+ ],
+ "p_stock_fp16": [
+ 0.9851218461990356,
+ 0.014878157526254654
+ ],
+ "p_fast": [
+ 0.9850931763648987,
+ 0.014906810596585274
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9777073860168457,
+ 0.02229265309870243
+ ],
+ "p_stock_fp16": [
+ 0.9776605367660522,
+ 0.022339442744851112
+ ],
+ "p_fast": [
+ 0.9777138829231262,
+ 0.0222861859947443
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9317946434020996,
+ 0.06820531189441681
+ ],
+ "p_stock_fp16": [
+ 0.9317269325256348,
+ 0.06827300041913986
+ ],
+ "p_fast": [
+ 0.9318356513977051,
+ 0.06816437840461731
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.6343872547149658,
+ 0.29902541637420654,
+ 0.045897550880908966,
+ 0.020689744502305984
+ ],
+ "p_stock_fp16": [
+ 0.6342507600784302,
+ 0.29916030168533325,
+ 0.045900121331214905,
+ 0.02068880759179592
+ ],
+ "p_fast": [
+ 0.6345157623291016,
+ 0.2988472282886505,
+ 0.04591929912567139,
+ 0.020717673003673553
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9825068116188049,
+ 0.01749311573803425
+ ],
+ "p_stock_fp16": [
+ 0.9825071692466736,
+ 0.017492765560746193
+ ],
+ "p_fast": [
+ 0.9824988842010498,
+ 0.017501160502433777
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9939368367195129,
+ 0.006063186097890139
+ ],
+ "p_stock_fp16": [
+ 0.9939349293708801,
+ 0.006065097637474537
+ ],
+ "p_fast": [
+ 0.9939378499984741,
+ 0.006062155123800039
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9386484026908875,
+ 0.06135157495737076
+ ],
+ "p_stock_fp16": [
+ 0.9385618567466736,
+ 0.06143811717629433
+ ],
+ "p_fast": [
+ 0.9384209513664246,
+ 0.06157904863357544
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6302990317344666,
+ 0.3697010278701782
+ ],
+ "p_stock_fp16": [
+ 0.6301729679107666,
+ 0.3698270320892334
+ ],
+ "p_fast": [
+ 0.6303009986877441,
+ 0.36969900131225586
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.3822575509548187,
+ 0.243504598736763,
+ 0.23485027253627777,
+ 0.1393875777721405
+ ],
+ "p_stock_fp16": [
+ 0.3818753957748413,
+ 0.24332810938358307,
+ 0.23497958481311798,
+ 0.13981688022613525
+ ],
+ "p_fast": [
+ 0.38158661127090454,
+ 0.24338169395923615,
+ 0.23485922813415527,
+ 0.14017239212989807
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.18079549074172974,
+ 0.8192045092582703
+ ],
+ "p_stock_fp16": [
+ 0.18086500465869904,
+ 0.8191349506378174
+ ],
+ "p_fast": [
+ 0.1808469146490097,
+ 0.8191531300544739
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.18861573934555054,
+ 0.8113842606544495
+ ],
+ "p_stock_fp16": [
+ 0.1886974573135376,
+ 0.8113025426864624
+ ],
+ "p_fast": [
+ 0.18852932751178741,
+ 0.8114706873893738
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6191400289535522,
+ 0.38085994124412537
+ ],
+ "p_stock_fp16": [
+ 0.6190107464790344,
+ 0.3809892237186432
+ ],
+ "p_fast": [
+ 0.6185212731361389,
+ 0.38147878646850586
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9660330414772034,
+ 0.03396691754460335
+ ],
+ "p_stock_fp16": [
+ 0.9660199284553528,
+ 0.03398000821471214
+ ],
+ "p_fast": [
+ 0.9659237265586853,
+ 0.03407631069421768
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.08258534967899323,
+ 0.2609964907169342,
+ 0.5500252842903137,
+ 0.10639287531375885
+ ],
+ "p_stock_fp16": [
+ 0.08252229541540146,
+ 0.26116907596588135,
+ 0.5499332547187805,
+ 0.10637542605400085
+ ],
+ "p_fast": [
+ 0.08264720439910889,
+ 0.261181503534317,
+ 0.5499594807624817,
+ 0.106211818754673
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9826089143753052,
+ 0.017391040921211243
+ ],
+ "p_stock_fp16": [
+ 0.9825909733772278,
+ 0.01740904338657856
+ ],
+ "p_fast": [
+ 0.9825825691223145,
+ 0.01741739548742771
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9797461628913879,
+ 0.020253824070096016
+ ],
+ "p_stock_fp16": [
+ 0.9798033833503723,
+ 0.020196635276079178
+ ],
+ "p_fast": [
+ 0.9797549247741699,
+ 0.020245004445314407
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9683158993721008,
+ 0.03168412670493126
+ ],
+ "p_stock_fp16": [
+ 0.9683662056922913,
+ 0.03163371607661247
+ ],
+ "p_fast": [
+ 0.9682763814926147,
+ 0.031723588705062866
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.16894322633743286,
+ 0.8310567736625671
+ ],
+ "p_stock_fp16": [
+ 0.1690969318151474,
+ 0.8309030532836914
+ ],
+ "p_fast": [
+ 0.1688055545091629,
+ 0.8311944603919983
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.27004456520080566,
+ 0.23373167216777802,
+ 0.34025830030441284,
+ 0.15596546232700348
+ ],
+ "p_stock_fp16": [
+ 0.27025413513183594,
+ 0.23377129435539246,
+ 0.34013521671295166,
+ 0.15583935379981995
+ ],
+ "p_fast": [
+ 0.2701181173324585,
+ 0.23376773297786713,
+ 0.3402961492538452,
+ 0.15581797063350677
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9679447412490845,
+ 0.03205527737736702
+ ],
+ "p_stock_fp16": [
+ 0.9678916931152344,
+ 0.03210829570889473
+ ],
+ "p_fast": [
+ 0.9678916931152344,
+ 0.03210829570889473
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9843999147415161,
+ 0.015600092709064484
+ ],
+ "p_stock_fp16": [
+ 0.9844114184379578,
+ 0.015588540583848953
+ ],
+ "p_fast": [
+ 0.9844114184379578,
+ 0.015588540583848953
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9328323602676392,
+ 0.06716759502887726
+ ],
+ "p_stock_fp16": [
+ 0.9330353140830994,
+ 0.06696470826864243
+ ],
+ "p_fast": [
+ 0.9331724047660828,
+ 0.06682755053043365
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8932691812515259,
+ 0.10673077404499054
+ ],
+ "p_stock_fp16": [
+ 0.8941903710365295,
+ 0.10580960661172867
+ ],
+ "p_fast": [
+ 0.8947988152503967,
+ 0.10520120710134506
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2749302089214325,
+ 0.17856481671333313,
+ 0.29601383209228516,
+ 0.2504911422729492
+ ],
+ "p_stock_fp16": [
+ 0.27445125579833984,
+ 0.1784362941980362,
+ 0.29628199338912964,
+ 0.2508304715156555
+ ],
+ "p_fast": [
+ 0.2754172086715698,
+ 0.17836621403694153,
+ 0.2959127128124237,
+ 0.25030383467674255
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9986273050308228,
+ 0.0013727452605962753
+ ],
+ "p_stock_fp16": [
+ 0.9986329674720764,
+ 0.001367025775834918
+ ],
+ "p_fast": [
+ 0.998620867729187,
+ 0.0013790767407044768
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9986945986747742,
+ 0.0013054306618869305
+ ],
+ "p_stock_fp16": [
+ 0.9987004995346069,
+ 0.001299427356570959
+ ],
+ "p_fast": [
+ 0.9986993074417114,
+ 0.00130069546867162
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9971616268157959,
+ 0.0028383766766637564
+ ],
+ "p_stock_fp16": [
+ 0.997164785861969,
+ 0.0028352299705147743
+ ],
+ "p_fast": [
+ 0.997164785861969,
+ 0.0028352299705147743
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9854265451431274,
+ 0.014573482796549797
+ ],
+ "p_stock_fp16": [
+ 0.985475480556488,
+ 0.014524532482028008
+ ],
+ "p_fast": [
+ 0.9854194521903992,
+ 0.014580551534891129
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5974613428115845,
+ 0.27034273743629456,
+ 0.10179945081472397,
+ 0.03039645589888096
+ ],
+ "p_stock_fp16": [
+ 0.5973610281944275,
+ 0.27004170417785645,
+ 0.10197211802005768,
+ 0.030625056475400925
+ ],
+ "p_fast": [
+ 0.5975050926208496,
+ 0.2702387571334839,
+ 0.10177284479141235,
+ 0.030483238399028778
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8529036045074463,
+ 0.14709636569023132
+ ],
+ "p_stock_fp16": [
+ 0.8524908423423767,
+ 0.1475091278553009
+ ],
+ "p_fast": [
+ 0.8529047966003418,
+ 0.14709514379501343
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9688681960105896,
+ 0.03113178350031376
+ ],
+ "p_stock_fp16": [
+ 0.9689003229141235,
+ 0.031099660322070122
+ ],
+ "p_fast": [
+ 0.9690471291542053,
+ 0.03095286525785923
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.965154230594635,
+ 0.03484576568007469
+ ],
+ "p_stock_fp16": [
+ 0.9650202989578247,
+ 0.034979645162820816
+ ],
+ "p_fast": [
+ 0.9651026725769043,
+ 0.034897331148386
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9680230617523193,
+ 0.03197691589593887
+ ],
+ "p_stock_fp16": [
+ 0.9679825901985168,
+ 0.03201737254858017
+ ],
+ "p_fast": [
+ 0.9680657386779785,
+ 0.03193424269556999
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.15638655424118042,
+ 0.2918219566345215,
+ 0.45086419582366943,
+ 0.10092726349830627
+ ],
+ "p_stock_fp16": [
+ 0.156502828001976,
+ 0.2918865382671356,
+ 0.450540155172348,
+ 0.10107050091028214
+ ],
+ "p_fast": [
+ 0.15643338859081268,
+ 0.29197075963020325,
+ 0.45089027285575867,
+ 0.10070551931858063
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.990936815738678,
+ 0.00906322244554758
+ ],
+ "p_stock_fp16": [
+ 0.990931510925293,
+ 0.009068417362868786
+ ],
+ "p_fast": [
+ 0.990922749042511,
+ 0.00907719787210226
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9939631819725037,
+ 0.006036823615431786
+ ],
+ "p_stock_fp16": [
+ 0.9937798380851746,
+ 0.0062200953252613544
+ ],
+ "p_fast": [
+ 0.9940021634101868,
+ 0.005997770465910435
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9746173024177551,
+ 0.025382665917277336
+ ],
+ "p_stock_fp16": [
+ 0.9744833707809448,
+ 0.025516601279377937
+ ],
+ "p_fast": [
+ 0.9745197892189026,
+ 0.025480203330516815
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6642186045646667,
+ 0.33578136563301086
+ ],
+ "p_stock_fp16": [
+ 0.6645267605781555,
+ 0.33547329902648926
+ ],
+ "p_fast": [
+ 0.6649619340896606,
+ 0.3350380063056946
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.45076873898506165,
+ 0.3377600908279419,
+ 0.16410930454730988,
+ 0.047361861914396286
+ ],
+ "p_stock_fp16": [
+ 0.45108142495155334,
+ 0.3376794457435608,
+ 0.1639270782470703,
+ 0.04731204733252525
+ ],
+ "p_fast": [
+ 0.4515349268913269,
+ 0.3368655741214752,
+ 0.16396158933639526,
+ 0.0476379357278347
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9556035399436951,
+ 0.04439644142985344
+ ],
+ "p_stock_fp16": [
+ 0.9555892944335938,
+ 0.04441070184111595
+ ],
+ "p_fast": [
+ 0.9556099772453308,
+ 0.044389985501766205
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9755018949508667,
+ 0.02449815534055233
+ ],
+ "p_stock_fp16": [
+ 0.9754835963249207,
+ 0.024516336619853973
+ ],
+ "p_fast": [
+ 0.9755303859710693,
+ 0.02446967177093029
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9696269035339355,
+ 0.030373062938451767
+ ],
+ "p_stock_fp16": [
+ 0.9696420431137085,
+ 0.03035798855125904
+ ],
+ "p_fast": [
+ 0.9696994423866272,
+ 0.030300550162792206
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9871689081192017,
+ 0.012831129133701324
+ ],
+ "p_stock_fp16": [
+ 0.9871540665626526,
+ 0.012845957651734352
+ ],
+ "p_fast": [
+ 0.9871788024902344,
+ 0.012821213342249393
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.3204658329486847,
+ 0.3182946741580963,
+ 0.20673127472400665,
+ 0.15450820326805115
+ ],
+ "p_stock_fp16": [
+ 0.3202156126499176,
+ 0.3182671070098877,
+ 0.2067089080810547,
+ 0.15480834245681763
+ ],
+ "p_fast": [
+ 0.3204858601093292,
+ 0.318535715341568,
+ 0.2066057324409485,
+ 0.15437264740467072
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9834228754043579,
+ 0.016577094793319702
+ ],
+ "p_stock_fp16": [
+ 0.9834227561950684,
+ 0.01657724939286709
+ ],
+ "p_fast": [
+ 0.9834227561950684,
+ 0.01657724939286709
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.996426522731781,
+ 0.0035734723787754774
+ ],
+ "p_stock_fp16": [
+ 0.996442973613739,
+ 0.0035570731852203608
+ ],
+ "p_fast": [
+ 0.9963994026184082,
+ 0.0036006032023578882
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9983080625534058,
+ 0.0016919723711907864
+ ],
+ "p_stock_fp16": [
+ 0.9983075857162476,
+ 0.0016924524679780006
+ ],
+ "p_fast": [
+ 0.9983084201812744,
+ 0.0016916276654228568
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9977206587791443,
+ 0.002279271837323904
+ ],
+ "p_stock_fp16": [
+ 0.9977327585220337,
+ 0.0022672584746032953
+ ],
+ "p_fast": [
+ 0.9977161884307861,
+ 0.0022838874720036983
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.6300764679908752,
+ 0.276387482881546,
+ 0.06863974034786224,
+ 0.024896353483200073
+ ],
+ "p_stock_fp16": [
+ 0.6299688220024109,
+ 0.2764250934123993,
+ 0.06867337971925735,
+ 0.024932648986577988
+ ],
+ "p_fast": [
+ 0.6300053000450134,
+ 0.2764410972595215,
+ 0.06864383816719055,
+ 0.02490975521504879
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9848158359527588,
+ 0.01518422830849886
+ ],
+ "p_stock_fp16": [
+ 0.9849128127098083,
+ 0.015087132342159748
+ ],
+ "p_fast": [
+ 0.9847817420959473,
+ 0.015218292362987995
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8343393206596375,
+ 0.16566069424152374
+ ],
+ "p_stock_fp16": [
+ 0.8344743251800537,
+ 0.1655256450176239
+ ],
+ "p_fast": [
+ 0.8346092104911804,
+ 0.16539080440998077
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7587581276893616,
+ 0.24124185740947723
+ ],
+ "p_stock_fp16": [
+ 0.7585776448249817,
+ 0.24142231047153473
+ ],
+ "p_fast": [
+ 0.7591584324836731,
+ 0.2408415526151657
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.03488408774137497,
+ 0.11611437797546387,
+ 0.387145459651947,
+ 0.46185603737831116
+ ],
+ "p_stock_fp16": [
+ 0.03493739292025566,
+ 0.11634509265422821,
+ 0.3871573507785797,
+ 0.4615601897239685
+ ],
+ "p_fast": [
+ 0.03501809015870094,
+ 0.11641468107700348,
+ 0.38729438185691833,
+ 0.46127286553382874
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.3391225039958954,
+ 0.04174385592341423,
+ 0.0269475094974041,
+ 0.42629510164260864,
+ 0.032247353345155716,
+ 0.13364359736442566
+ ],
+ "p_stock_fp16": [
+ 0.3393939435482025,
+ 0.04180112108588219,
+ 0.026988836005330086,
+ 0.4259032905101776,
+ 0.032206907868385315,
+ 0.13370594382286072
+ ],
+ "p_fast": [
+ 0.33949539065361023,
+ 0.04185447096824646,
+ 0.027023278176784515,
+ 0.42561474442481995,
+ 0.03231105953454971,
+ 0.13370102643966675
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9927472472190857,
+ 0.007252735085785389
+ ],
+ "p_stock_fp16": [
+ 0.9927074313163757,
+ 0.007292611990123987
+ ],
+ "p_fast": [
+ 0.9928370118141174,
+ 0.007162978406995535
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8958684206008911,
+ 0.10413163155317307
+ ],
+ "p_stock_fp16": [
+ 0.8960127830505371,
+ 0.10398723185062408
+ ],
+ "p_fast": [
+ 0.8956710696220398,
+ 0.1043289378285408
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6366130113601685,
+ 0.36338695883750916
+ ],
+ "p_stock_fp16": [
+ 0.6367908120155334,
+ 0.36320915818214417
+ ],
+ "p_fast": [
+ 0.63619065284729,
+ 0.36380934715270996
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0338304229080677,
+ 0.06095995754003525,
+ 0.30135539174079895,
+ 0.603854238986969
+ ],
+ "p_stock_fp16": [
+ 0.03387724980711937,
+ 0.06104497238993645,
+ 0.30164989829063416,
+ 0.6034278869628906
+ ],
+ "p_fast": [
+ 0.03387724980711937,
+ 0.06104497238993645,
+ 0.30164989829063416,
+ 0.6034278869628906
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.2601044774055481,
+ 0.31739547848701477,
+ 0.01272143330425024,
+ 0.028285974636673927,
+ 0.3531803786754608,
+ 0.028312217444181442
+ ],
+ "p_stock_fp16": [
+ 0.2600550651550293,
+ 0.31800442934036255,
+ 0.012721777893602848,
+ 0.028252072632312775,
+ 0.3526870012283325,
+ 0.028279680758714676
+ ],
+ "p_fast": [
+ 0.2605750560760498,
+ 0.31801852583885193,
+ 0.01274721510708332,
+ 0.0283639058470726,
+ 0.352014422416687,
+ 0.02828093245625496
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9817904233932495,
+ 0.01820952072739601
+ ],
+ "p_stock_fp16": [
+ 0.9817968606948853,
+ 0.018203092738986015
+ ],
+ "p_fast": [
+ 0.9818925857543945,
+ 0.018107349053025246
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9153319597244263,
+ 0.08466808497905731
+ ],
+ "p_stock_fp16": [
+ 0.915280282497406,
+ 0.08471965044736862
+ ],
+ "p_fast": [
+ 0.9153182506561279,
+ 0.08468180149793625
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9010818600654602,
+ 0.09891817718744278
+ ],
+ "p_stock_fp16": [
+ 0.9013360142707825,
+ 0.09866397827863693
+ ],
+ "p_fast": [
+ 0.9010969400405884,
+ 0.09890306740999222
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5014544129371643,
+ 0.3993522524833679,
+ 0.07377302646636963,
+ 0.025420313701033592
+ ],
+ "p_stock_fp16": [
+ 0.5015986561775208,
+ 0.3991290032863617,
+ 0.07383144646883011,
+ 0.025440821424126625
+ ],
+ "p_fast": [
+ 0.5008786916732788,
+ 0.39972546696662903,
+ 0.07394178211688995,
+ 0.025453969836235046
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.21425652503967285,
+ 0.03020607866346836,
+ 0.1317135989665985,
+ 0.24805848300457,
+ 0.03104798123240471,
+ 0.34471726417541504
+ ],
+ "p_stock_fp16": [
+ 0.2142544686794281,
+ 0.030221274122595787,
+ 0.13175685703754425,
+ 0.24799399077892303,
+ 0.031089385971426964,
+ 0.34468403458595276
+ ],
+ "p_fast": [
+ 0.21421495079994202,
+ 0.03025999665260315,
+ 0.13150762021541595,
+ 0.2481905072927475,
+ 0.031038152053952217,
+ 0.3447887897491455
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.18380992114543915,
+ 0.8161900639533997
+ ],
+ "p_stock_fp16": [
+ 0.18427157402038574,
+ 0.8157284259796143
+ ],
+ "p_fast": [
+ 0.183831587433815,
+ 0.8161683678627014
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.23025445640087128,
+ 0.7697455286979675
+ ],
+ "p_stock_fp16": [
+ 0.23035657405853271,
+ 0.7696433663368225
+ ],
+ "p_fast": [
+ 0.2298159897327423,
+ 0.7701840400695801
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9547452926635742,
+ 0.04525475576519966
+ ],
+ "p_stock_fp16": [
+ 0.9545839428901672,
+ 0.04541604593396187
+ ],
+ "p_fast": [
+ 0.9546685814857483,
+ 0.0453314445912838
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.20860610902309418,
+ 0.18134969472885132,
+ 0.29079294204711914,
+ 0.3192512094974518
+ ],
+ "p_stock_fp16": [
+ 0.20895208418369293,
+ 0.1813192516565323,
+ 0.29088136553764343,
+ 0.31884732842445374
+ ],
+ "p_fast": [
+ 0.20873978734016418,
+ 0.18122349679470062,
+ 0.29065677523612976,
+ 0.31937992572784424
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.15682639181613922,
+ 0.2380664348602295,
+ 0.10981667786836624,
+ 0.2613714933395386,
+ 0.18578195571899414,
+ 0.04813706874847412
+ ],
+ "p_stock_fp16": [
+ 0.15669117867946625,
+ 0.23834291100502014,
+ 0.1100577712059021,
+ 0.26125770807266235,
+ 0.18544046580791473,
+ 0.04820997267961502
+ ],
+ "p_fast": [
+ 0.15683673322200775,
+ 0.2380988448858261,
+ 0.10997189581394196,
+ 0.26150038838386536,
+ 0.18543154001235962,
+ 0.048160601407289505
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8540359735488892,
+ 0.14596398174762726
+ ],
+ "p_stock_fp16": [
+ 0.8541790246963501,
+ 0.1458209604024887
+ ],
+ "p_fast": [
+ 0.8541714549064636,
+ 0.14582857489585876
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8814943432807922,
+ 0.11850562691688538
+ ],
+ "p_stock_fp16": [
+ 0.881347119808197,
+ 0.11865291744470596
+ ],
+ "p_fast": [
+ 0.881793200969696,
+ 0.11820685863494873
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.967627763748169,
+ 0.03237223997712135
+ ],
+ "p_stock_fp16": [
+ 0.967625081539154,
+ 0.03237491101026535
+ ],
+ "p_fast": [
+ 0.9677014350891113,
+ 0.03229851648211479
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.06822484731674194,
+ 0.13737277686595917,
+ 0.39298364520072937,
+ 0.4014187157154083
+ ],
+ "p_stock_fp16": [
+ 0.06801412254571915,
+ 0.13722050189971924,
+ 0.3935021162033081,
+ 0.4012632369995117
+ ],
+ "p_fast": [
+ 0.06809563934803009,
+ 0.1373472511768341,
+ 0.39320501685142517,
+ 0.40135204792022705
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.11639373004436493,
+ 0.12373795360326767,
+ 0.17377381026744843,
+ 0.16929568350315094,
+ 0.07642018795013428,
+ 0.34037864208221436
+ ],
+ "p_stock_fp16": [
+ 0.11631379276514053,
+ 0.1236945316195488,
+ 0.17409688234329224,
+ 0.16874049603939056,
+ 0.07620598375797272,
+ 0.3409483730792999
+ ],
+ "p_fast": [
+ 0.11648242175579071,
+ 0.12363215535879135,
+ 0.17366954684257507,
+ 0.16948093473911285,
+ 0.07654038071632385,
+ 0.34019455313682556
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.08370641618967056,
+ 0.9162935614585876
+ ],
+ "p_stock_fp16": [
+ 0.0832919254899025,
+ 0.9167081117630005
+ ],
+ "p_fast": [
+ 0.08387162536382675,
+ 0.9161283373832703
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3785058557987213,
+ 0.6214941740036011
+ ],
+ "p_stock_fp16": [
+ 0.37937837839126587,
+ 0.6206215620040894
+ ],
+ "p_fast": [
+ 0.3777773678302765,
+ 0.6222226619720459
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7780634164810181,
+ 0.22193655371665955
+ ],
+ "p_stock_fp16": [
+ 0.7780807018280029,
+ 0.22191926836967468
+ ],
+ "p_fast": [
+ 0.7780306339263916,
+ 0.22196932137012482
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.20972195267677307,
+ 0.24862968921661377,
+ 0.35670939087867737,
+ 0.1849389374256134
+ ],
+ "p_stock_fp16": [
+ 0.20955106616020203,
+ 0.24862033128738403,
+ 0.3565671741962433,
+ 0.18526144325733185
+ ],
+ "p_fast": [
+ 0.20969218015670776,
+ 0.24866628646850586,
+ 0.35645899176597595,
+ 0.1851826161146164
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.23745842278003693,
+ 0.09393908828496933,
+ 0.03363722562789917,
+ 0.22656786441802979,
+ 0.23654960095882416,
+ 0.17184776067733765
+ ],
+ "p_stock_fp16": [
+ 0.23708942532539368,
+ 0.09390559047460556,
+ 0.03363053500652313,
+ 0.226951465010643,
+ 0.2365979254245758,
+ 0.17182506620883942
+ ],
+ "p_fast": [
+ 0.23761674761772156,
+ 0.09381622076034546,
+ 0.03364778682589531,
+ 0.2264312505722046,
+ 0.23645932972431183,
+ 0.17202864587306976
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3019067049026489,
+ 0.6980932950973511
+ ],
+ "p_stock_fp16": [
+ 0.30195352435112,
+ 0.6980465054512024
+ ],
+ "p_fast": [
+ 0.3072151243686676,
+ 0.6927849054336548
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.659529983997345,
+ 0.34046998620033264
+ ],
+ "p_stock_fp16": [
+ 0.659021258354187,
+ 0.3409787714481354
+ ],
+ "p_fast": [
+ 0.6592269539833069,
+ 0.3407730758190155
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.4156767725944519,
+ 0.5843231678009033
+ ],
+ "p_stock_fp16": [
+ 0.4158548414707184,
+ 0.5841451287269592
+ ],
+ "p_fast": [
+ 0.4151729643344879,
+ 0.5848270058631897
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.08883614838123322,
+ 0.13282839953899384,
+ 0.2743772268295288,
+ 0.5039582848548889
+ ],
+ "p_stock_fp16": [
+ 0.0891495943069458,
+ 0.13300693035125732,
+ 0.274131178855896,
+ 0.5037122964859009
+ ],
+ "p_fast": [
+ 0.08882921934127808,
+ 0.13287723064422607,
+ 0.27455005049705505,
+ 0.5037434697151184
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.14637786149978638,
+ 0.3235582113265991,
+ 0.03658990561962128,
+ 0.04097359627485275,
+ 0.4122118353843689,
+ 0.04028867930173874
+ ],
+ "p_stock_fp16": [
+ 0.1466779261827469,
+ 0.32354024052619934,
+ 0.036674343049526215,
+ 0.04107331857085228,
+ 0.4115976095199585,
+ 0.04043653607368469
+ ],
+ "p_fast": [
+ 0.14630569517612457,
+ 0.32366108894348145,
+ 0.03661645948886871,
+ 0.04100848734378815,
+ 0.4121536612510681,
+ 0.04025460034608841
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7632602453231812,
+ 0.23673976957798004
+ ],
+ "p_stock_fp16": [
+ 0.7666596174240112,
+ 0.23334036767482758
+ ],
+ "p_fast": [
+ 0.7659217715263367,
+ 0.23407818377017975
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8913726210594177,
+ 0.10862734913825989
+ ],
+ "p_stock_fp16": [
+ 0.8911793828010559,
+ 0.10882068425416946
+ ],
+ "p_fast": [
+ 0.8913390040397644,
+ 0.1086609736084938
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8998916149139404,
+ 0.10010836273431778
+ ],
+ "p_stock_fp16": [
+ 0.8998965620994568,
+ 0.1001034751534462
+ ],
+ "p_fast": [
+ 0.899673581123352,
+ 0.10032637417316437
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.46722984313964844,
+ 0.369934618473053,
+ 0.11892163753509521,
+ 0.04391384497284889
+ ],
+ "p_stock_fp16": [
+ 0.46722686290740967,
+ 0.3699679970741272,
+ 0.119005486369133,
+ 0.04379971697926521
+ ],
+ "p_fast": [
+ 0.46729162335395813,
+ 0.37001925706863403,
+ 0.11884051561355591,
+ 0.043848589062690735
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.19406408071517944,
+ 0.2018871158361435,
+ 0.26675018668174744,
+ 0.09229136258363724,
+ 0.042288582772016525,
+ 0.20271864533424377
+ ],
+ "p_stock_fp16": [
+ 0.19438979029655457,
+ 0.20139452815055847,
+ 0.26647892594337463,
+ 0.09222764521837234,
+ 0.04216308519244194,
+ 0.20334607362747192
+ ],
+ "p_fast": [
+ 0.19383105635643005,
+ 0.20178629457950592,
+ 0.2669972777366638,
+ 0.09233938902616501,
+ 0.04217296838760376,
+ 0.20287300646305084
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9983421564102173,
+ 0.001657813205383718
+ ],
+ "p_stock_fp16": [
+ 0.9983354210853577,
+ 0.001664633397012949
+ ],
+ "p_fast": [
+ 0.9983345866203308,
+ 0.0016654449282214046
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9729209542274475,
+ 0.027079008519649506
+ ],
+ "p_stock_fp16": [
+ 0.972933828830719,
+ 0.027066176757216454
+ ],
+ "p_fast": [
+ 0.9728372097015381,
+ 0.02716277912259102
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5183061361312866,
+ 0.4816938638687134
+ ],
+ "p_stock_fp16": [
+ 0.5183632969856262,
+ 0.4816366732120514
+ ],
+ "p_fast": [
+ 0.517989993095398,
+ 0.48201000690460205
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.023643840104341507,
+ 0.15977902710437775,
+ 0.25296124815940857,
+ 0.5636158585548401
+ ],
+ "p_stock_fp16": [
+ 0.023550482466816902,
+ 0.15890808403491974,
+ 0.25195828080177307,
+ 0.5655831098556519
+ ],
+ "p_fast": [
+ 0.02361370250582695,
+ 0.15941251814365387,
+ 0.2526346445083618,
+ 0.5643391013145447
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.28367963433265686,
+ 0.05565132573246956,
+ 0.05322139710187912,
+ 0.11245077848434448,
+ 0.22624066472053528,
+ 0.2687561810016632
+ ],
+ "p_stock_fp16": [
+ 0.2838732898235321,
+ 0.05557139590382576,
+ 0.05333820730447769,
+ 0.11258666962385178,
+ 0.22528985142707825,
+ 0.2693406045436859
+ ],
+ "p_fast": [
+ 0.2841900885105133,
+ 0.05583752319216728,
+ 0.053124673664569855,
+ 0.11241002380847931,
+ 0.22591325640678406,
+ 0.2685243785381317
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9274953007698059,
+ 0.07250470668077469
+ ],
+ "p_stock_fp16": [
+ 0.9276177883148193,
+ 0.07238221913576126
+ ],
+ "p_fast": [
+ 0.9275686144828796,
+ 0.07243140786886215
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9180071353912354,
+ 0.08199288696050644
+ ],
+ "p_stock_fp16": [
+ 0.918022096157074,
+ 0.08197791874408722
+ ],
+ "p_fast": [
+ 0.9186628460884094,
+ 0.08133713901042938
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8884492516517639,
+ 0.11155081540346146
+ ],
+ "p_stock_fp16": [
+ 0.8885293006896973,
+ 0.11147071421146393
+ ],
+ "p_fast": [
+ 0.8883357048034668,
+ 0.11166432499885559
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2410876750946045,
+ 0.35780009627342224,
+ 0.19815438985824585,
+ 0.20295782387256622
+ ],
+ "p_stock_fp16": [
+ 0.24078142642974854,
+ 0.35741913318634033,
+ 0.19854553043842316,
+ 0.20325392484664917
+ ],
+ "p_fast": [
+ 0.24140851199626923,
+ 0.3580002188682556,
+ 0.1978996843099594,
+ 0.20269165933132172
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.08929187059402466,
+ 0.20355252921581268,
+ 0.1244087740778923,
+ 0.04064306989312172,
+ 0.1756647676229477,
+ 0.36643898487091064
+ ],
+ "p_stock_fp16": [
+ 0.08945275098085403,
+ 0.20293666422367096,
+ 0.12455722689628601,
+ 0.04067544639110565,
+ 0.1755681037902832,
+ 0.36680978536605835
+ ],
+ "p_fast": [
+ 0.0892782136797905,
+ 0.2026519775390625,
+ 0.12461806833744049,
+ 0.040615908801555634,
+ 0.17584693431854248,
+ 0.3669889271259308
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.984460175037384,
+ 0.015539758838713169
+ ],
+ "p_stock_fp16": [
+ 0.9844264388084412,
+ 0.015573563054203987
+ ],
+ "p_fast": [
+ 0.9844637513160706,
+ 0.015536176972091198
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8587174415588379,
+ 0.14128261804580688
+ ],
+ "p_stock_fp16": [
+ 0.858065664768219,
+ 0.14193430542945862
+ ],
+ "p_fast": [
+ 0.8591622710227966,
+ 0.14083771407604218
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6365188360214233,
+ 0.36348119378089905
+ ],
+ "p_stock_fp16": [
+ 0.6366020441055298,
+ 0.3633979856967926
+ ],
+ "p_fast": [
+ 0.6360193490982056,
+ 0.36398062109947205
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.04678545519709587,
+ 0.1299351155757904,
+ 0.2839876115322113,
+ 0.5392918586730957
+ ],
+ "p_stock_fp16": [
+ 0.04678943008184433,
+ 0.1299489438533783,
+ 0.28411194682121277,
+ 0.5391497015953064
+ ],
+ "p_fast": [
+ 0.04678656905889511,
+ 0.12987756729125977,
+ 0.2839558720588684,
+ 0.5393800139427185
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.24041950702667236,
+ 0.16486990451812744,
+ 0.04929640144109726,
+ 0.038499459624290466,
+ 0.40653443336486816,
+ 0.1003803089261055
+ ],
+ "p_stock_fp16": [
+ 0.24048259854316711,
+ 0.1649988740682602,
+ 0.04933645948767662,
+ 0.03853600472211838,
+ 0.4062877893447876,
+ 0.1003582626581192
+ ],
+ "p_fast": [
+ 0.24010159075260162,
+ 0.16465705633163452,
+ 0.049306418746709824,
+ 0.0385313518345356,
+ 0.40703293681144714,
+ 0.10037063807249069
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9987101554870605,
+ 0.0012898908462375402
+ ],
+ "p_stock_fp16": [
+ 0.9987094402313232,
+ 0.0012905863113701344
+ ],
+ "p_fast": [
+ 0.9986985921859741,
+ 0.0013013298157602549
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9770605564117432,
+ 0.022939473390579224
+ ],
+ "p_stock_fp16": [
+ 0.977022647857666,
+ 0.022977368906140327
+ ],
+ "p_fast": [
+ 0.9770335555076599,
+ 0.0229664109647274
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9678712487220764,
+ 0.0321287140250206
+ ],
+ "p_stock_fp16": [
+ 0.967808187007904,
+ 0.03219185769557953
+ ],
+ "p_fast": [
+ 0.9678993225097656,
+ 0.032100703567266464
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.4592909812927246,
+ 0.36317652463912964,
+ 0.10236763209104538,
+ 0.07516488432884216
+ ],
+ "p_stock_fp16": [
+ 0.45945432782173157,
+ 0.3634583055973053,
+ 0.10219325870275497,
+ 0.07489407062530518
+ ],
+ "p_fast": [
+ 0.4594138264656067,
+ 0.3630715012550354,
+ 0.10233405232429504,
+ 0.07518056780099869
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.23899833858013153,
+ 0.13979384303092957,
+ 0.14536887407302856,
+ 0.07561320066452026,
+ 0.07931211590766907,
+ 0.3209136724472046
+ ],
+ "p_stock_fp16": [
+ 0.23872976005077362,
+ 0.14003396034240723,
+ 0.14525721967220306,
+ 0.07569035142660141,
+ 0.07940030843019485,
+ 0.3208884000778198
+ ],
+ "p_fast": [
+ 0.23868133127689362,
+ 0.13956192135810852,
+ 0.14554719626903534,
+ 0.0755457729101181,
+ 0.0793260708451271,
+ 0.3213377296924591
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.15363943576812744,
+ 0.5411243438720703,
+ 0.20558011531829834,
+ 0.09965616464614868
+ ],
+ "p_stock_fp16": [
+ 0.15386320650577545,
+ 0.5409831404685974,
+ 0.2054719626903534,
+ 0.09968167543411255
+ ],
+ "p_fast": [
+ 0.15378591418266296,
+ 0.5409754514694214,
+ 0.20560702681541443,
+ 0.09963160008192062
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.14609380066394806,
+ 0.22429321706295013,
+ 0.24897204339504242,
+ 0.14454080164432526,
+ 0.06140107661485672,
+ 0.17469905316829681
+ ],
+ "p_stock_fp16": [
+ 0.14579595625400543,
+ 0.2247132807970047,
+ 0.24995242059230804,
+ 0.14452014863491058,
+ 0.06137323006987572,
+ 0.17364497482776642
+ ],
+ "p_fast": [
+ 0.14633595943450928,
+ 0.22357195615768433,
+ 0.24941252171993256,
+ 0.1443488746881485,
+ 0.06136039271950722,
+ 0.17497026920318604
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9460921287536621,
+ 0.05390787124633789
+ ],
+ "p_stock_fp16": [
+ 0.9463369250297546,
+ 0.05366310477256775
+ ],
+ "p_fast": [
+ 0.946026086807251,
+ 0.05397389829158783
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5182129740715027,
+ 0.4817869961261749
+ ],
+ "p_stock_fp16": [
+ 0.5176875591278076,
+ 0.48231241106987
+ ],
+ "p_fast": [
+ 0.5180135369300842,
+ 0.48198646306991577
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.061001695692539215,
+ 0.2587133049964905,
+ 0.3934936225414276,
+ 0.2867914140224457
+ ],
+ "p_stock_fp16": [
+ 0.06124548614025116,
+ 0.25898876786231995,
+ 0.3933711051940918,
+ 0.2863946259021759
+ ],
+ "p_fast": [
+ 0.06105390936136246,
+ 0.25814715027809143,
+ 0.39386770129203796,
+ 0.28693124651908875
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9106506705284119,
+ 0.01608418859541416,
+ 0.021707722917199135,
+ 0.017790555953979492,
+ 0.014671149663627148,
+ 0.01909572072327137
+ ],
+ "p_stock_fp16": [
+ 0.9108679294586182,
+ 0.01605968363583088,
+ 0.021610666066408157,
+ 0.017776448279619217,
+ 0.014651095494627953,
+ 0.01903413236141205
+ ],
+ "p_fast": [
+ 0.9109857082366943,
+ 0.01603041961789131,
+ 0.021613461896777153,
+ 0.017709434032440186,
+ 0.0146243991330266,
+ 0.01903659477829933
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.47418609261512756,
+ 0.5258139371871948
+ ],
+ "p_stock_fp16": [
+ 0.4736039638519287,
+ 0.5263960361480713
+ ],
+ "p_fast": [
+ 0.4741365611553192,
+ 0.5258634090423584
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6362482905387878,
+ 0.36375170946121216
+ ],
+ "p_stock_fp16": [
+ 0.6369495987892151,
+ 0.3630503714084625
+ ],
+ "p_fast": [
+ 0.6362683773040771,
+ 0.36373162269592285
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.11406733840703964,
+ 0.6427571177482605,
+ 0.19504785537719727,
+ 0.048127733170986176
+ ],
+ "p_stock_fp16": [
+ 0.11424713581800461,
+ 0.642982006072998,
+ 0.19471493363380432,
+ 0.0480559840798378
+ ],
+ "p_fast": [
+ 0.11419608443975449,
+ 0.6428123712539673,
+ 0.19490131735801697,
+ 0.04809024557471275
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.06784193217754364,
+ 0.07607923448085785,
+ 0.1005159467458725,
+ 0.09539924561977386,
+ 0.05125369876623154,
+ 0.6089099049568176
+ ],
+ "p_stock_fp16": [
+ 0.06779628247022629,
+ 0.0762254074215889,
+ 0.10039229691028595,
+ 0.09523534029722214,
+ 0.05127536505460739,
+ 0.6090753078460693
+ ],
+ "p_fast": [
+ 0.06771513819694519,
+ 0.07605987042188644,
+ 0.10076294839382172,
+ 0.09530733525753021,
+ 0.05121399834752083,
+ 0.6089407205581665
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9281124472618103,
+ 0.07188758999109268
+ ],
+ "p_stock_fp16": [
+ 0.9283925890922546,
+ 0.07160742580890656
+ ],
+ "p_fast": [
+ 0.9281486868858337,
+ 0.07185126841068268
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.901957094669342,
+ 0.09804288297891617
+ ],
+ "p_stock_fp16": [
+ 0.901942253112793,
+ 0.09805773198604584
+ ],
+ "p_fast": [
+ 0.9019530415534973,
+ 0.0980469286441803
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2526102364063263,
+ 0.5600556135177612,
+ 0.15408360958099365,
+ 0.03325054049491882
+ ],
+ "p_stock_fp16": [
+ 0.2528931498527527,
+ 0.5599731802940369,
+ 0.15391263365745544,
+ 0.033220987766981125
+ ],
+ "p_fast": [
+ 0.25295791029930115,
+ 0.5595698952674866,
+ 0.1541777104139328,
+ 0.03329446166753769
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.29658329486846924,
+ 0.21880574524402618,
+ 0.18251341581344604,
+ 0.12181292474269867,
+ 0.09264285117387772,
+ 0.08764180541038513
+ ],
+ "p_stock_fp16": [
+ 0.2962764501571655,
+ 0.21891435980796814,
+ 0.18264207243919373,
+ 0.12178471684455872,
+ 0.09292075783014297,
+ 0.08746162801980972
+ ],
+ "p_fast": [
+ 0.29627862572669983,
+ 0.21867559850215912,
+ 0.1827995479106903,
+ 0.12188972532749176,
+ 0.09281940758228302,
+ 0.08753704279661179
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9349043965339661,
+ 0.06509559601545334
+ ],
+ "p_stock_fp16": [
+ 0.9349318742752075,
+ 0.06506811827421188
+ ],
+ "p_fast": [
+ 0.9348130226135254,
+ 0.06518702954053879
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.978571355342865,
+ 0.021428629755973816
+ ],
+ "p_stock_fp16": [
+ 0.9785903096199036,
+ 0.02140977419912815
+ ],
+ "p_fast": [
+ 0.9785903096199036,
+ 0.02140977419912815
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.3465142548084259,
+ 0.5596334338188171,
+ 0.05834595486521721,
+ 0.03550642356276512
+ ],
+ "p_stock_fp16": [
+ 0.3466929793357849,
+ 0.5594505667686462,
+ 0.05833562836050987,
+ 0.035520829260349274
+ ],
+ "p_fast": [
+ 0.34636127948760986,
+ 0.5597346425056458,
+ 0.05836524814367294,
+ 0.035538867115974426
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.0395582914352417,
+ 0.17612579464912415,
+ 0.14405174553394318,
+ 0.16424626111984253,
+ 0.03720676153898239,
+ 0.4388110935688019
+ ],
+ "p_stock_fp16": [
+ 0.03955981135368347,
+ 0.17625896632671356,
+ 0.1435776799917221,
+ 0.16461262106895447,
+ 0.03723566234111786,
+ 0.43875524401664734
+ ],
+ "p_fast": [
+ 0.03961864113807678,
+ 0.17600469291210175,
+ 0.14407230913639069,
+ 0.1645357459783554,
+ 0.03721826896071434,
+ 0.43855032324790955
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8548730611801147,
+ 0.14512692391872406
+ ],
+ "p_stock_fp16": [
+ 0.8549869060516357,
+ 0.14501316845417023
+ ],
+ "p_fast": [
+ 0.8551343679428101,
+ 0.14486567676067352
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9914765954017639,
+ 0.008523386903107166
+ ],
+ "p_stock_fp16": [
+ 0.9914349913597107,
+ 0.00856503751128912
+ ],
+ "p_fast": [
+ 0.991468071937561,
+ 0.008531930856406689
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.18917544186115265,
+ 0.5456979870796204,
+ 0.18691468238830566,
+ 0.07821187376976013
+ ],
+ "p_stock_fp16": [
+ 0.18936008214950562,
+ 0.5459286570549011,
+ 0.1866748034954071,
+ 0.07803643494844437
+ ],
+ "p_fast": [
+ 0.1892174482345581,
+ 0.5457838773727417,
+ 0.18683047592639923,
+ 0.07816825807094574
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.04277270659804344,
+ 0.07839668542146683,
+ 0.0593654103577137,
+ 0.06592525541782379,
+ 0.060230057686567307,
+ 0.6933098435401917
+ ],
+ "p_stock_fp16": [
+ 0.042760029435157776,
+ 0.07841765880584717,
+ 0.05936641991138458,
+ 0.06584110856056213,
+ 0.06012490391731262,
+ 0.6934898495674133
+ ],
+ "p_fast": [
+ 0.042825810611248016,
+ 0.07838504761457443,
+ 0.059399716556072235,
+ 0.06574948877096176,
+ 0.060099903494119644,
+ 0.6935400366783142
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6870533227920532,
+ 0.31294670701026917
+ ],
+ "p_stock_fp16": [
+ 0.6876693367958069,
+ 0.3123306930065155
+ ],
+ "p_fast": [
+ 0.6870659589767456,
+ 0.312934011220932
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3506668508052826,
+ 0.6493331789970398
+ ],
+ "p_stock_fp16": [
+ 0.34976866841316223,
+ 0.6502313017845154
+ ],
+ "p_fast": [
+ 0.3499075174331665,
+ 0.6500925421714783
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.1377384066581726,
+ 0.4004904329776764,
+ 0.2280898541212082,
+ 0.233681321144104
+ ],
+ "p_stock_fp16": [
+ 0.13792778551578522,
+ 0.40114957094192505,
+ 0.22773262858390808,
+ 0.23319004476070404
+ ],
+ "p_fast": [
+ 0.13835939764976501,
+ 0.4008452594280243,
+ 0.2277822047472,
+ 0.23301313817501068
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.626227617263794,
+ 0.12996284663677216,
+ 0.11615025252103806,
+ 0.05005913972854614,
+ 0.03461715579032898,
+ 0.0429830364882946
+ ],
+ "p_stock_fp16": [
+ 0.6266649961471558,
+ 0.12995260953903198,
+ 0.11592136323451996,
+ 0.04995463415980339,
+ 0.03456879034638405,
+ 0.04293760657310486
+ ],
+ "p_fast": [
+ 0.625145673751831,
+ 0.13027210533618927,
+ 0.11671814322471619,
+ 0.05017535015940666,
+ 0.03468763083219528,
+ 0.04300115257501602
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9017393589019775,
+ 0.09826064109802246
+ ],
+ "p_stock_fp16": [
+ 0.9016395211219788,
+ 0.09836044162511826
+ ],
+ "p_fast": [
+ 0.9020286202430725,
+ 0.09797139465808868
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9524552822113037,
+ 0.0475446842610836
+ ],
+ "p_stock_fp16": [
+ 0.9529805183410645,
+ 0.04701944440603256
+ ],
+ "p_fast": [
+ 0.9530789256095886,
+ 0.04692108929157257
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0750429704785347,
+ 0.42906996607780457,
+ 0.3325490951538086,
+ 0.16333794593811035
+ ],
+ "p_stock_fp16": [
+ 0.07499916106462479,
+ 0.42917367815971375,
+ 0.33261269330978394,
+ 0.16321441531181335
+ ],
+ "p_fast": [
+ 0.07523348182439804,
+ 0.42956966161727905,
+ 0.3322700262069702,
+ 0.16292686760425568
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.06006060540676117,
+ 0.35301893949508667,
+ 0.11654307693243027,
+ 0.2826351225376129,
+ 0.06683231890201569,
+ 0.12090993672609329
+ ],
+ "p_stock_fp16": [
+ 0.06013423949480057,
+ 0.352873831987381,
+ 0.11636544018983841,
+ 0.2831270694732666,
+ 0.06682305037975311,
+ 0.12067639827728271
+ ],
+ "p_fast": [
+ 0.06014484912157059,
+ 0.35397160053253174,
+ 0.116073839366436,
+ 0.2823486030101776,
+ 0.06670442968606949,
+ 0.12075663357973099
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9648438096046448,
+ 0.03515619784593582
+ ],
+ "p_stock_fp16": [
+ 0.9649956822395325,
+ 0.0350043810904026
+ ],
+ "p_fast": [
+ 0.9649626016616821,
+ 0.03503737971186638
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8355194926261902,
+ 0.1644805520772934
+ ],
+ "p_stock_fp16": [
+ 0.8370546698570251,
+ 0.16294534504413605
+ ],
+ "p_fast": [
+ 0.8357518315315247,
+ 0.16424818336963654
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.01463442575186491,
+ 0.15192097425460815,
+ 0.582795262336731,
+ 0.2506493330001831
+ ],
+ "p_stock_fp16": [
+ 0.01460236869752407,
+ 0.15180768072605133,
+ 0.5827202796936035,
+ 0.2508697211742401
+ ],
+ "p_fast": [
+ 0.014587132260203362,
+ 0.1511872112751007,
+ 0.5832502841949463,
+ 0.2509753406047821
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.32057663798332214,
+ 0.10239028930664062,
+ 0.12426351010799408,
+ 0.1029951274394989,
+ 0.04367315396666527,
+ 0.30610138177871704
+ ],
+ "p_stock_fp16": [
+ 0.320098876953125,
+ 0.1024097204208374,
+ 0.12413433194160461,
+ 0.10291097313165665,
+ 0.04366043955087662,
+ 0.3067857027053833
+ ],
+ "p_fast": [
+ 0.32030534744262695,
+ 0.10232576727867126,
+ 0.12403256446123123,
+ 0.10302765667438507,
+ 0.043624650686979294,
+ 0.3066839575767517
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8969318270683289,
+ 0.10306819528341293
+ ],
+ "p_stock_fp16": [
+ 0.8967835903167725,
+ 0.10321635007858276
+ ],
+ "p_fast": [
+ 0.8971953392028809,
+ 0.10280467569828033
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8312171697616577,
+ 0.16878283023834229
+ ],
+ "p_stock_fp16": [
+ 0.8314939141273499,
+ 0.16850602626800537
+ ],
+ "p_fast": [
+ 0.8315324187278748,
+ 0.16846755146980286
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.17582300305366516,
+ 0.5501419305801392,
+ 0.18685142695903778,
+ 0.08718368411064148
+ ],
+ "p_stock_fp16": [
+ 0.17619574069976807,
+ 0.551267683506012,
+ 0.1859181523323059,
+ 0.08661841601133347
+ ],
+ "p_fast": [
+ 0.1758798211812973,
+ 0.5509514212608337,
+ 0.18635663390159607,
+ 0.08681212365627289
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.17655961215496063,
+ 0.0788390040397644,
+ 0.27396804094314575,
+ 0.13127033412456512,
+ 0.04060972481966019,
+ 0.2987532913684845
+ ],
+ "p_stock_fp16": [
+ 0.17706924676895142,
+ 0.0789971649646759,
+ 0.2730143368244171,
+ 0.13139407336711884,
+ 0.040664251893758774,
+ 0.29886093735694885
+ ],
+ "p_fast": [
+ 0.17815808951854706,
+ 0.07874895632266998,
+ 0.2733551263809204,
+ 0.13110920786857605,
+ 0.0406157411634922,
+ 0.2980128526687622
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9413192868232727,
+ 0.05868071690201759
+ ],
+ "p_stock_fp16": [
+ 0.9413987994194031,
+ 0.05860121548175812
+ ],
+ "p_fast": [
+ 0.9415870904922485,
+ 0.05841294303536415
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9728504419326782,
+ 0.027149531990289688
+ ],
+ "p_stock_fp16": [
+ 0.9727919697761536,
+ 0.02720797434449196
+ ],
+ "p_fast": [
+ 0.972908079624176,
+ 0.027091901749372482
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0297623872756958,
+ 0.1067776158452034,
+ 0.4541211724281311,
+ 0.4093388319015503
+ ],
+ "p_stock_fp16": [
+ 0.029771875590085983,
+ 0.10680769383907318,
+ 0.4540346562862396,
+ 0.4093857705593109
+ ],
+ "p_fast": [
+ 0.02977488376200199,
+ 0.10671747475862503,
+ 0.4540805220603943,
+ 0.40942710638046265
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.6785348057746887,
+ 0.12510277330875397,
+ 0.041234564036130905,
+ 0.04308178648352623,
+ 0.07209224253892899,
+ 0.039953723549842834
+ ],
+ "p_stock_fp16": [
+ 0.6791352033615112,
+ 0.12471061199903488,
+ 0.04116537794470787,
+ 0.04301474243402481,
+ 0.07203613221645355,
+ 0.039937831461429596
+ ],
+ "p_fast": [
+ 0.6781370043754578,
+ 0.12519796192646027,
+ 0.04130607098340988,
+ 0.04311962053179741,
+ 0.0722823366522789,
+ 0.03995709866285324
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5614522695541382,
+ 0.4385477602481842
+ ],
+ "p_stock_fp16": [
+ 0.5611246228218079,
+ 0.4388754069805145
+ ],
+ "p_fast": [
+ 0.5613651275634766,
+ 0.43863487243652344
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9627551436424255,
+ 0.03724486753344536
+ ],
+ "p_stock_fp16": [
+ 0.9627432227134705,
+ 0.037256766110658646
+ ],
+ "p_fast": [
+ 0.9627782702445984,
+ 0.037221759557724
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.08946849405765533,
+ 0.30836138129234314,
+ 0.4054504334926605,
+ 0.1967197209596634
+ ],
+ "p_stock_fp16": [
+ 0.08956188708543777,
+ 0.3085828423500061,
+ 0.4052285850048065,
+ 0.1966266632080078
+ ],
+ "p_fast": [
+ 0.08952055126428604,
+ 0.308365136384964,
+ 0.405338317155838,
+ 0.19677594304084778
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.14360684156417847,
+ 0.23243853449821472,
+ 0.1422262340784073,
+ 0.07544311136007309,
+ 0.06089730188250542,
+ 0.34538793563842773
+ ],
+ "p_stock_fp16": [
+ 0.1426790952682495,
+ 0.23272459208965302,
+ 0.14226168394088745,
+ 0.0754808634519577,
+ 0.060887955129146576,
+ 0.3459658622741699
+ ],
+ "p_fast": [
+ 0.1429613083600998,
+ 0.2327299416065216,
+ 0.1422649621963501,
+ 0.0754089206457138,
+ 0.060829926282167435,
+ 0.34580492973327637
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9559788107872009,
+ 0.04402123764157295
+ ],
+ "p_stock_fp16": [
+ 0.9559608697891235,
+ 0.044039200991392136
+ ],
+ "p_fast": [
+ 0.956063449382782,
+ 0.043936531990766525
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9810004234313965,
+ 0.018999604508280754
+ ],
+ "p_stock_fp16": [
+ 0.9810037016868591,
+ 0.0189962275326252
+ ],
+ "p_fast": [
+ 0.9810037016868591,
+ 0.0189962275326252
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.8675079345703125,
+ 0.01611124537885189,
+ 0.023573242127895355,
+ 0.039238445460796356,
+ 0.017879746854305267,
+ 0.03568948805332184
+ ],
+ "p_stock_fp16": [
+ 0.8672703504562378,
+ 0.016150519251823425,
+ 0.02359084039926529,
+ 0.039314787834882736,
+ 0.017876997590065002,
+ 0.035796523094177246
+ ],
+ "p_fast": [
+ 0.867561399936676,
+ 0.016140170395374298,
+ 0.02364489622414112,
+ 0.03913641721010208,
+ 0.017882997170090675,
+ 0.0356341116130352
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7418201565742493,
+ 0.2581798732280731
+ ],
+ "p_stock_fp16": [
+ 0.7414049506187439,
+ 0.2585950195789337
+ ],
+ "p_fast": [
+ 0.7418435215950012,
+ 0.2581564486026764
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.707365870475769,
+ 0.29263415932655334
+ ],
+ "p_stock_fp16": [
+ 0.7077709436416626,
+ 0.2922290563583374
+ ],
+ "p_fast": [
+ 0.7073794603347778,
+ 0.29262056946754456
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.006834388244897127,
+ 0.3354489803314209,
+ 0.657716691493988
+ ],
+ "p_stock_fp16": [
+ 0.006847100798040628,
+ 0.33576229214668274,
+ 0.6573905944824219
+ ],
+ "p_fast": [
+ 0.006825059652328491,
+ 0.33533576130867004,
+ 0.6578391790390015
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8669513463973999,
+ 0.1330486387014389
+ ],
+ "p_stock_fp16": [
+ 0.8667117357254028,
+ 0.13328824937343597
+ ],
+ "p_fast": [
+ 0.8666834831237793,
+ 0.13331645727157593
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.006087132263928652,
+ 0.7426608204841614,
+ 0.007194834761321545,
+ 0.19574464857578278,
+ 0.008362685330212116,
+ 0.03994989022612572
+ ],
+ "p_stock_fp16": [
+ 0.006093243602663279,
+ 0.7424198389053345,
+ 0.007207695860415697,
+ 0.19595474004745483,
+ 0.008377419784665108,
+ 0.03994692116975784
+ ],
+ "p_fast": [
+ 0.006087535060942173,
+ 0.7431744933128357,
+ 0.007200943306088448,
+ 0.19529378414154053,
+ 0.008353239856660366,
+ 0.03989001736044884
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6881608366966248,
+ 0.31183910369873047
+ ],
+ "p_stock_fp16": [
+ 0.6884946227073669,
+ 0.31150537729263306
+ ],
+ "p_fast": [
+ 0.6893709897994995,
+ 0.3106290102005005
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7670397758483887,
+ 0.23296020925045013
+ ],
+ "p_stock_fp16": [
+ 0.766992449760437,
+ 0.2330075204372406
+ ],
+ "p_fast": [
+ 0.7677551507949829,
+ 0.2322448492050171
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.0042326608672738075,
+ 0.1721258908510208,
+ 0.8236414790153503
+ ],
+ "p_stock_fp16": [
+ 0.0042305151000618935,
+ 0.1720680594444275,
+ 0.8237013816833496
+ ],
+ "p_fast": [
+ 0.004231225699186325,
+ 0.1719290018081665,
+ 0.8238397836685181
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6243698596954346,
+ 0.3756301701068878
+ ],
+ "p_stock_fp16": [
+ 0.6251574754714966,
+ 0.3748425543308258
+ ],
+ "p_fast": [
+ 0.6242094039916992,
+ 0.3757905960083008
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.11010496318340302,
+ 0.0191094558686018,
+ 0.48997482657432556,
+ 0.021628309041261673,
+ 0.034384146332740784,
+ 0.3247982859611511
+ ],
+ "p_stock_fp16": [
+ 0.1098865196108818,
+ 0.019118739292025566,
+ 0.48971959948539734,
+ 0.021622097119688988,
+ 0.03438371419906616,
+ 0.32526928186416626
+ ],
+ "p_fast": [
+ 0.11002665013074875,
+ 0.019129103049635887,
+ 0.48998507857322693,
+ 0.021612701937556267,
+ 0.03443596512079239,
+ 0.32481053471565247
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9472465515136719,
+ 0.052753400057554245
+ ],
+ "p_stock_fp16": [
+ 0.9473323225975037,
+ 0.052667658776044846
+ ],
+ "p_fast": [
+ 0.9473323225975037,
+ 0.052667658776044846
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9539501070976257,
+ 0.046049848198890686
+ ],
+ "p_stock_fp16": [
+ 0.9538156986236572,
+ 0.0461842343211174
+ ],
+ "p_fast": [
+ 0.954008936882019,
+ 0.04599103331565857
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.5677735209465027,
+ 0.3914264142513275,
+ 0.0408000610768795
+ ],
+ "p_stock_fp16": [
+ 0.5671040415763855,
+ 0.39205503463745117,
+ 0.04084087163209915
+ ],
+ "p_fast": [
+ 0.5680062174797058,
+ 0.39114782214164734,
+ 0.04084596782922745
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5804012417793274,
+ 0.4195987582206726
+ ],
+ "p_stock_fp16": [
+ 0.5806344747543335,
+ 0.4193655252456665
+ ],
+ "p_fast": [
+ 0.5804598331451416,
+ 0.4195401668548584
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.02506846934556961,
+ 0.08374807238578796,
+ 0.03477940708398819,
+ 0.13404075801372528,
+ 0.06331031024456024,
+ 0.6590529680252075
+ ],
+ "p_stock_fp16": [
+ 0.02505299635231495,
+ 0.08384762704372406,
+ 0.034782636910676956,
+ 0.1340944766998291,
+ 0.06335334479808807,
+ 0.6588689088821411
+ ],
+ "p_fast": [
+ 0.02504829503595829,
+ 0.08379098027944565,
+ 0.03467437997460365,
+ 0.13384857773780823,
+ 0.06324874609708786,
+ 0.6593889594078064
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5329158306121826,
+ 0.46708419919013977
+ ],
+ "p_stock_fp16": [
+ 0.5328201651573181,
+ 0.4671798348426819
+ ],
+ "p_fast": [
+ 0.5334886312484741,
+ 0.46651139855384827
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6222484707832336,
+ 0.37775152921676636
+ ],
+ "p_stock_fp16": [
+ 0.6215983033180237,
+ 0.37840166687965393
+ ],
+ "p_fast": [
+ 0.622201144695282,
+ 0.3777988851070404
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.2441190630197525,
+ 0.41501176357269287,
+ 0.3408691883087158
+ ],
+ "p_stock_fp16": [
+ 0.24467770755290985,
+ 0.41479063034057617,
+ 0.34053170680999756
+ ],
+ "p_fast": [
+ 0.2436952441930771,
+ 0.41514724493026733,
+ 0.3411574959754944
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8240371942520142,
+ 0.17596276104450226
+ ],
+ "p_stock_fp16": [
+ 0.8229596614837646,
+ 0.17704035341739655
+ ],
+ "p_fast": [
+ 0.8238936066627502,
+ 0.17610633373260498
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.02197432890534401,
+ 0.03848302364349365,
+ 0.018150469288229942,
+ 0.042898982763290405,
+ 0.05884347856044769,
+ 0.8196496963500977
+ ],
+ "p_stock_fp16": [
+ 0.02195354923605919,
+ 0.03845449909567833,
+ 0.01816461607813835,
+ 0.042773593217134476,
+ 0.05886559933423996,
+ 0.8197881579399109
+ ],
+ "p_fast": [
+ 0.021957190707325935,
+ 0.038460876792669296,
+ 0.018132181838154793,
+ 0.04282248765230179,
+ 0.05870312824845314,
+ 0.8199241757392883
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6074861884117126,
+ 0.39251381158828735
+ ],
+ "p_stock_fp16": [
+ 0.6064984202384949,
+ 0.39350154995918274
+ ],
+ "p_fast": [
+ 0.6068770885467529,
+ 0.3931228816509247
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8024953007698059,
+ 0.1975046694278717
+ ],
+ "p_stock_fp16": [
+ 0.8023233413696289,
+ 0.19767668843269348
+ ],
+ "p_fast": [
+ 0.8030194044113159,
+ 0.19698064029216766
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.07369204610586166,
+ 0.6068594455718994,
+ 0.3194485306739807
+ ],
+ "p_stock_fp16": [
+ 0.07362333685159683,
+ 0.6068835854530334,
+ 0.31949302554130554
+ ],
+ "p_fast": [
+ 0.07386830449104309,
+ 0.6068252325057983,
+ 0.3193063735961914
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.42933106422424316,
+ 0.5706689953804016
+ ],
+ "p_stock_fp16": [
+ 0.42847251892089844,
+ 0.5715274810791016
+ ],
+ "p_fast": [
+ 0.4290406107902527,
+ 0.5709594488143921
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.03612571209669113,
+ 0.04793607071042061,
+ 0.1972629874944687,
+ 0.59276282787323,
+ 0.017160922288894653,
+ 0.10875151306390762
+ ],
+ "p_stock_fp16": [
+ 0.03602796047925949,
+ 0.047916099429130554,
+ 0.19696494936943054,
+ 0.5935094356536865,
+ 0.017151881009340286,
+ 0.1084296926856041
+ ],
+ "p_fast": [
+ 0.03604357689619064,
+ 0.04793687164783478,
+ 0.19714656472206116,
+ 0.593187153339386,
+ 0.017176080495119095,
+ 0.10850979387760162
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.13375742733478546,
+ 0.866242527961731
+ ],
+ "p_stock_fp16": [
+ 0.1336977332830429,
+ 0.8663023114204407
+ ],
+ "p_fast": [
+ 0.1336977332830429,
+ 0.8663023114204407
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.18096666038036346,
+ 0.8190333247184753
+ ],
+ "p_stock_fp16": [
+ 0.18097351491451263,
+ 0.8190264105796814
+ ],
+ "p_fast": [
+ 0.18137192726135254,
+ 0.8186280727386475
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.06192639842629433,
+ 0.5973776578903198,
+ 0.34069591760635376
+ ],
+ "p_stock_fp16": [
+ 0.062041811645030975,
+ 0.5976141095161438,
+ 0.34034404158592224
+ ],
+ "p_fast": [
+ 0.06219436973333359,
+ 0.597622811794281,
+ 0.3401828408241272
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6549398899078369,
+ 0.3450601100921631
+ ],
+ "p_stock_fp16": [
+ 0.654949963092804,
+ 0.3450500965118408
+ ],
+ "p_fast": [
+ 0.6546739935874939,
+ 0.3453260362148285
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.025907106697559357,
+ 0.2321919947862625,
+ 0.018995536491274834,
+ 0.30921074748039246,
+ 0.10555917769670486,
+ 0.308135449886322
+ ],
+ "p_stock_fp16": [
+ 0.025964904576539993,
+ 0.23145093023777008,
+ 0.019052064046263695,
+ 0.3085375130176544,
+ 0.10600477457046509,
+ 0.30898982286453247
+ ],
+ "p_fast": [
+ 0.026036983355879784,
+ 0.23144271969795227,
+ 0.019049065187573433,
+ 0.3080373704433441,
+ 0.10603984445333481,
+ 0.3093940317630768
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.48461663722991943,
+ 0.5153834223747253
+ ],
+ "p_stock_fp16": [
+ 0.4853670597076416,
+ 0.5146329402923584
+ ],
+ "p_fast": [
+ 0.4825415015220642,
+ 0.5174584984779358
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5290076732635498,
+ 0.4709923565387726
+ ],
+ "p_stock_fp16": [
+ 0.5293850302696228,
+ 0.4706149697303772
+ ],
+ "p_fast": [
+ 0.5277120471000671,
+ 0.47228798270225525
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.008156470023095608,
+ 0.3449278473854065,
+ 0.6469156742095947
+ ],
+ "p_stock_fp16": [
+ 0.008109756745398045,
+ 0.3444990813732147,
+ 0.6473912000656128
+ ],
+ "p_fast": [
+ 0.008136093616485596,
+ 0.34427037835121155,
+ 0.6475935578346252
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.0917402058839798,
+ 0.9082598686218262
+ ],
+ "p_stock_fp16": [
+ 0.09199207276105881,
+ 0.9080079793930054
+ ],
+ "p_fast": [
+ 0.09148351103067398,
+ 0.9085164666175842
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.5252329111099243,
+ 0.022251885384321213,
+ 0.12854264676570892,
+ 0.017590103670954704,
+ 0.03141535073518753,
+ 0.27496716380119324
+ ],
+ "p_stock_fp16": [
+ 0.524016261100769,
+ 0.022337106987833977,
+ 0.12896552681922913,
+ 0.017601218074560165,
+ 0.03134693577885628,
+ 0.27573296427726746
+ ],
+ "p_fast": [
+ 0.5254661440849304,
+ 0.022333383560180664,
+ 0.1286139190196991,
+ 0.017632687464356422,
+ 0.031341709196567535,
+ 0.2746121883392334
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9589805006980896,
+ 0.041019506752491
+ ],
+ "p_stock_fp16": [
+ 0.958895206451416,
+ 0.041104745119810104
+ ],
+ "p_fast": [
+ 0.9591638445854187,
+ 0.040836151689291
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9364396929740906,
+ 0.06356034427881241
+ ],
+ "p_stock_fp16": [
+ 0.9362558722496033,
+ 0.06374412775039673
+ ],
+ "p_fast": [
+ 0.9363723993301392,
+ 0.06362766772508621
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.24989286065101624,
+ 0.572486162185669,
+ 0.1776210069656372
+ ],
+ "p_stock_fp16": [
+ 0.24986889958381653,
+ 0.5725144147872925,
+ 0.177616685628891
+ ],
+ "p_fast": [
+ 0.25000861287117004,
+ 0.5722753405570984,
+ 0.1777159869670868
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.23296482861042023,
+ 0.767035186290741
+ ],
+ "p_stock_fp16": [
+ 0.23344413936138153,
+ 0.766555905342102
+ ],
+ "p_fast": [
+ 0.23340044915676117,
+ 0.7665995955467224
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.12097876518964767,
+ 0.22702394425868988,
+ 0.050346773117780685,
+ 0.07463028281927109,
+ 0.23697327077388763,
+ 0.29004696011543274
+ ],
+ "p_stock_fp16": [
+ 0.12049388885498047,
+ 0.22779186069965363,
+ 0.05027840659022331,
+ 0.0743066668510437,
+ 0.23616769909858704,
+ 0.29096147418022156
+ ],
+ "p_fast": [
+ 0.12109403312206268,
+ 0.22776441276073456,
+ 0.05040561035275459,
+ 0.07493244111537933,
+ 0.23559756577014923,
+ 0.2902059257030487
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8749845623970032,
+ 0.1250154972076416
+ ],
+ "p_stock_fp16": [
+ 0.8752123713493347,
+ 0.12478765845298767
+ ],
+ "p_fast": [
+ 0.8748577833175659,
+ 0.12514221668243408
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6822959780693054,
+ 0.3177039921283722
+ ],
+ "p_stock_fp16": [
+ 0.6819651126861572,
+ 0.3180348873138428
+ ],
+ "p_fast": [
+ 0.6829042434692383,
+ 0.3170957565307617
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.004992176312953234,
+ 0.4028315842151642,
+ 0.5921761989593506
+ ],
+ "p_stock_fp16": [
+ 0.00498608686029911,
+ 0.40272462368011475,
+ 0.5922892689704895
+ ],
+ "p_fast": [
+ 0.004982276353985071,
+ 0.4020240008831024,
+ 0.5929936766624451
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7262336015701294,
+ 0.2737663686275482
+ ],
+ "p_stock_fp16": [
+ 0.7255331873893738,
+ 0.2744668126106262
+ ],
+ "p_fast": [
+ 0.7279331684112549,
+ 0.2720668613910675
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.073619544506073,
+ 0.07772894203662872,
+ 0.05636807158589363,
+ 0.11553928256034851,
+ 0.042433951050043106,
+ 0.634310245513916
+ ],
+ "p_stock_fp16": [
+ 0.07381094247102737,
+ 0.07773183286190033,
+ 0.05631715804338455,
+ 0.11549889296293259,
+ 0.04242747277021408,
+ 0.6342137455940247
+ ],
+ "p_fast": [
+ 0.07361898571252823,
+ 0.07775714993476868,
+ 0.056280504912137985,
+ 0.11525477468967438,
+ 0.042358480393886566,
+ 0.6347301006317139
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.39480140805244446,
+ 0.6051985621452332
+ ],
+ "p_stock_fp16": [
+ 0.3961411714553833,
+ 0.6038588881492615
+ ],
+ "p_fast": [
+ 0.39519253373146057,
+ 0.604807436466217
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7475924491882324,
+ 0.2524075210094452
+ ],
+ "p_stock_fp16": [
+ 0.7477425932884216,
+ 0.25225740671157837
+ ],
+ "p_fast": [
+ 0.7468551397323608,
+ 0.25314489006996155
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.13149239122867584,
+ 0.46978768706321716,
+ 0.39871999621391296
+ ],
+ "p_stock_fp16": [
+ 0.1316521316766739,
+ 0.46960482001304626,
+ 0.3987430930137634
+ ],
+ "p_fast": [
+ 0.13142132759094238,
+ 0.46878156065940857,
+ 0.39979714155197144
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7810052633285522,
+ 0.21899472177028656
+ ],
+ "p_stock_fp16": [
+ 0.7813201546669006,
+ 0.21867981553077698
+ ],
+ "p_fast": [
+ 0.7811428308486938,
+ 0.21885713934898376
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.006574548780918121,
+ 0.875615656375885,
+ 0.007916916161775589,
+ 0.028746768832206726,
+ 0.014641680754721165,
+ 0.06650436669588089
+ ],
+ "p_stock_fp16": [
+ 0.006581632886081934,
+ 0.8755987286567688,
+ 0.007923474535346031,
+ 0.028813540935516357,
+ 0.014659140259027481,
+ 0.06642347574234009
+ ],
+ "p_fast": [
+ 0.006593658123165369,
+ 0.8754869103431702,
+ 0.007937951013445854,
+ 0.028866183012723923,
+ 0.014700273051857948,
+ 0.06641498953104019
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9809960126876831,
+ 0.01900395005941391
+ ],
+ "p_stock_fp16": [
+ 0.9810037016868591,
+ 0.0189962275326252
+ ],
+ "p_fast": [
+ 0.9810583591461182,
+ 0.018941707909107208
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9931678175926208,
+ 0.006832200102508068
+ ],
+ "p_stock_fp16": [
+ 0.9931893348693848,
+ 0.0068107303231954575
+ ],
+ "p_fast": [
+ 0.9931495785713196,
+ 0.006850480567663908
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.01595943048596382,
+ 0.36753395199775696,
+ 0.6165065765380859
+ ],
+ "p_stock_fp16": [
+ 0.01596658304333687,
+ 0.3676813542842865,
+ 0.6163520216941833
+ ],
+ "p_fast": [
+ 0.0159761942923069,
+ 0.3679026961326599,
+ 0.6161210536956787
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.44522884488105774,
+ 0.5547711253166199
+ ],
+ "p_stock_fp16": [
+ 0.4447908401489258,
+ 0.5552091598510742
+ ],
+ "p_fast": [
+ 0.44525817036628723,
+ 0.5547418594360352
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9162148833274841,
+ 0.013147835619747639,
+ 0.015526391565799713,
+ 0.0067847673781216145,
+ 0.008260440081357956,
+ 0.04006573557853699
+ ],
+ "p_stock_fp16": [
+ 0.9162437915802002,
+ 0.01314632035791874,
+ 0.015535599552094936,
+ 0.006793631240725517,
+ 0.008258955553174019,
+ 0.040021754801273346
+ ],
+ "p_fast": [
+ 0.9162036776542664,
+ 0.013171444647014141,
+ 0.015550095587968826,
+ 0.006780078634619713,
+ 0.008274740539491177,
+ 0.04002000391483307
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9984169006347656,
+ 0.0015830977354198694
+ ],
+ "p_stock_fp16": [
+ 0.9984083771705627,
+ 0.0015916259726509452
+ ],
+ "p_fast": [
+ 0.9984076619148254,
+ 0.0015924021136015654
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9993016719818115,
+ 0.0006983057828620076
+ ],
+ "p_stock_fp16": [
+ 0.9992951154708862,
+ 0.0007048390689305961
+ ],
+ "p_fast": [
+ 0.9992978572845459,
+ 0.0007020931225270033
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.29234760999679565,
+ 0.47026416659355164,
+ 0.23738817870616913
+ ],
+ "p_stock_fp16": [
+ 0.2920960783958435,
+ 0.47019943594932556,
+ 0.23770445585250854
+ ],
+ "p_fast": [
+ 0.29273682832717896,
+ 0.4700818657875061,
+ 0.23718130588531494
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7040170431137085,
+ 0.2959829270839691
+ ],
+ "p_stock_fp16": [
+ 0.7037721872329712,
+ 0.2962278127670288
+ ],
+ "p_fast": [
+ 0.7045859098434448,
+ 0.29541411995887756
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/benchmarks/results/parity_typed_decisions_rtx4070.json b/benchmarks/results/parity_typed_decisions_rtx4070.json
new file mode 100644
index 00000000..332e5944
--- /dev/null
+++ b/benchmarks/results/parity_typed_decisions_rtx4070.json
@@ -0,0 +1,6122 @@
+{
+ "model": "convaiinnovations/laya-typed-decisions",
+ "subfolder": null,
+ "gpu": "NVIDIA GeForce RTX 4070 Ti SUPER",
+ "torch": "2.11.0+cu130",
+ "dtype": "torch.bfloat16",
+ "summary": {
+ "choice": {
+ "n": 48,
+ "d_fast_stock": 0.03192096948623657,
+ "d_fast_fp32": 0.019468963146209717,
+ "d_stock_fp32": 0.015756309032440186,
+ "agree_fast_stock": 46,
+ "agree_fast_fp32": 47,
+ "agree_stock_fp32": 47
+ },
+ "noul": {
+ "n": 180,
+ "d_fast_stock": 0.041225239634513855,
+ "d_fast_fp32": 0.022663414478302002,
+ "d_stock_fp32": 0.030285969376564026,
+ "agree_fast_stock": 180,
+ "agree_fast_fp32": 180,
+ "agree_stock_fp32": 180
+ },
+ "score": {
+ "n": 60,
+ "d_fast_stock": 0.008714377880096436,
+ "d_fast_fp32": 0.009223639965057373,
+ "d_stock_fp32": 0.015246719121932983,
+ "agree_fast_stock": 60,
+ "agree_fast_fp32": 60,
+ "agree_stock_fp32": 60
+ }
+ },
+ "cases": [
+ {
+ "state": "triage/0",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.6927211284637451,
+ 0.014246546663343906,
+ 0.13677994906902313,
+ 0.007951270788908005,
+ 0.10968707501888275,
+ 0.038614001125097275
+ ],
+ "p_stock_bf16": [
+ 0.6927706003189087,
+ 0.014266105368733406,
+ 0.13748452067375183,
+ 0.007940283045172691,
+ 0.10875921696424484,
+ 0.03877928853034973
+ ],
+ "p_fast": [
+ 0.689109206199646,
+ 0.014302005060017109,
+ 0.13891153037548065,
+ 0.007898317649960518,
+ 0.11074993014335632,
+ 0.039029043167829514
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.4151865839958191,
+ 0.5848134160041809
+ ],
+ "p_stock_bf16": [
+ 0.4134771525859833,
+ 0.5865228176116943
+ ],
+ "p_fast": [
+ 0.41673731803894043,
+ 0.5832627415657043
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.02600688301026821,
+ 0.15357312560081482,
+ 0.5514219403266907,
+ 0.26899799704551697
+ ],
+ "p_stock_bf16": [
+ 0.025435276329517365,
+ 0.15279631316661835,
+ 0.5567267537117004,
+ 0.26504167914390564
+ ],
+ "p_fast": [
+ 0.026104522868990898,
+ 0.15438544750213623,
+ 0.5537958741188049,
+ 0.265714168548584
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3149517774581909,
+ 0.6850481629371643
+ ],
+ "p_stock_bf16": [
+ 0.31489264965057373,
+ 0.6851073503494263
+ ],
+ "p_fast": [
+ 0.31289464235305786,
+ 0.6871053576469421
+ ]
+ },
+ {
+ "state": "triage/0",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.16822032630443573,
+ 0.8317797183990479
+ ],
+ "p_stock_bf16": [
+ 0.17078442871570587,
+ 0.8292155861854553
+ ],
+ "p_fast": [
+ 0.16491955518722534,
+ 0.8350804448127747
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.006520497612655163,
+ 0.9102498888969421,
+ 0.008270232938230038,
+ 0.009572864510118961,
+ 0.00717336218804121,
+ 0.05821319296956062
+ ],
+ "p_stock_bf16": [
+ 0.0066267517395317554,
+ 0.9095861315727234,
+ 0.008376997895538807,
+ 0.009641867130994797,
+ 0.0073926751501858234,
+ 0.058375485241413116
+ ],
+ "p_fast": [
+ 0.00662142364308238,
+ 0.9088547825813293,
+ 0.008370263502001762,
+ 0.009634114801883698,
+ 0.007272210903465748,
+ 0.05924708768725395
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.20784282684326172,
+ 0.7921571731567383
+ ],
+ "p_stock_bf16": [
+ 0.2075357586145401,
+ 0.7924642562866211
+ ],
+ "p_fast": [
+ 0.20882359147071838,
+ 0.791176438331604
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.035688433796167374,
+ 0.2066349983215332,
+ 0.46579089760780334,
+ 0.29188570380210876
+ ],
+ "p_stock_bf16": [
+ 0.035617273300886154,
+ 0.2098238170146942,
+ 0.46551448106765747,
+ 0.28904449939727783
+ ],
+ "p_fast": [
+ 0.03514186292886734,
+ 0.20864683389663696,
+ 0.4665338397026062,
+ 0.2896774411201477
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9487811923027039,
+ 0.05121885612607002
+ ],
+ "p_stock_bf16": [
+ 0.9487276077270508,
+ 0.051272422075271606
+ ],
+ "p_fast": [
+ 0.9509603977203369,
+ 0.04903962463140488
+ ]
+ },
+ {
+ "state": "triage/1",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9140059947967529,
+ 0.08599407225847244
+ ],
+ "p_stock_bf16": [
+ 0.9130584001541138,
+ 0.08694165199995041
+ ],
+ "p_fast": [
+ 0.9107043743133545,
+ 0.0892956405878067
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.029065648093819618,
+ 0.027133196592330933,
+ 0.21221938729286194,
+ 0.3385237455368042,
+ 0.022455455735325813,
+ 0.370602548122406
+ ],
+ "p_stock_bf16": [
+ 0.02853900007903576,
+ 0.027232101187109947,
+ 0.21253018081188202,
+ 0.3350105583667755,
+ 0.02222619205713272,
+ 0.37446197867393494
+ ],
+ "p_fast": [
+ 0.028686219826340675,
+ 0.027372583746910095,
+ 0.2096993327140808,
+ 0.33477145433425903,
+ 0.02234085090458393,
+ 0.37712955474853516
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9442138075828552,
+ 0.0557861328125
+ ],
+ "p_stock_bf16": [
+ 0.9436599016189575,
+ 0.05634009465575218
+ ],
+ "p_fast": [
+ 0.9436599016189575,
+ 0.05634009465575218
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.6581481695175171,
+ 0.2644989490509033,
+ 0.05550163611769676,
+ 0.02185126207768917
+ ],
+ "p_stock_bf16": [
+ 0.6599099636077881,
+ 0.2635214626789093,
+ 0.0550217479467392,
+ 0.021546823903918266
+ ],
+ "p_fast": [
+ 0.6628720164299011,
+ 0.2616203725337982,
+ 0.054199717938899994,
+ 0.021307986229658127
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9795986413955688,
+ 0.020401325076818466
+ ],
+ "p_stock_bf16": [
+ 0.9795897603034973,
+ 0.020410310477018356
+ ],
+ "p_fast": [
+ 0.9796676635742188,
+ 0.020332353189587593
+ ]
+ },
+ {
+ "state": "triage/2",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9609938859939575,
+ 0.039006058126688004
+ ],
+ "p_stock_bf16": [
+ 0.9608049392700195,
+ 0.03919513151049614
+ ],
+ "p_fast": [
+ 0.9606574773788452,
+ 0.039342500269412994
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.011759733781218529,
+ 0.10597094893455505,
+ 0.027097292244434357,
+ 0.1957862675189972,
+ 0.09853699803352356,
+ 0.5608487725257874
+ ],
+ "p_stock_bf16": [
+ 0.011558075435459614,
+ 0.10281500220298767,
+ 0.02666420117020607,
+ 0.1928354948759079,
+ 0.09602148085832596,
+ 0.5701057314872742
+ ],
+ "p_fast": [
+ 0.011680805124342442,
+ 0.10431339591741562,
+ 0.026529552415013313,
+ 0.1914873719215393,
+ 0.09876195341348648,
+ 0.5672268867492676
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9739807844161987,
+ 0.026019226759672165
+ ],
+ "p_stock_bf16": [
+ 0.9737446904182434,
+ 0.026255298405885696
+ ],
+ "p_fast": [
+ 0.9737446904182434,
+ 0.026255298405885696
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.20646154880523682,
+ 0.22206242382526398,
+ 0.4460499584674835,
+ 0.1254260540008545
+ ],
+ "p_stock_bf16": [
+ 0.2051946520805359,
+ 0.2205720841884613,
+ 0.44818630814552307,
+ 0.12604694068431854
+ ],
+ "p_fast": [
+ 0.20234663784503937,
+ 0.22223427891731262,
+ 0.4480496644973755,
+ 0.12736940383911133
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9999996423721313,
+ 3.830248829217453e-07
+ ],
+ "p_stock_bf16": [
+ 0.9999997615814209,
+ 2.884932825963915e-07
+ ],
+ "p_fast": [
+ 0.9999996423721313,
+ 3.4798929959833913e-07
+ ]
+ },
+ {
+ "state": "triage/3",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9140607714653015,
+ 0.08593921363353729
+ ],
+ "p_stock_bf16": [
+ 0.9146726131439209,
+ 0.08532741665840149
+ ],
+ "p_fast": [
+ 0.9155827760696411,
+ 0.08441724628210068
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.051845304667949677,
+ 0.021178212016820908,
+ 0.01883244886994362,
+ 0.025343209505081177,
+ 0.04202444478869438,
+ 0.8407763242721558
+ ],
+ "p_stock_bf16": [
+ 0.05277344956994057,
+ 0.02148963324725628,
+ 0.018964534625411034,
+ 0.02551957219839096,
+ 0.042737238109111786,
+ 0.83851557970047
+ ],
+ "p_fast": [
+ 0.05105289816856384,
+ 0.020789017900824547,
+ 0.018635155633091927,
+ 0.024687565863132477,
+ 0.041343893855810165,
+ 0.8434914946556091
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9925909042358398,
+ 0.007409045938402414
+ ],
+ "p_stock_bf16": [
+ 0.9926256537437439,
+ 0.007374365348368883
+ ],
+ "p_fast": [
+ 0.9925682544708252,
+ 0.007431773468852043
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.01001009251922369,
+ 0.023612383753061295,
+ 0.21541395783424377,
+ 0.750963568687439
+ ],
+ "p_stock_bf16": [
+ 0.009867101907730103,
+ 0.023669946938753128,
+ 0.21851669251918793,
+ 0.7479462623596191
+ ],
+ "p_fast": [
+ 0.009919139556586742,
+ 0.023609604686498642,
+ 0.2145804911851883,
+ 0.7518907785415649
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9891748428344727,
+ 0.010825135745108128
+ ],
+ "p_stock_bf16": [
+ 0.9891815781593323,
+ 0.010818447917699814
+ ],
+ "p_fast": [
+ 0.9895108342170715,
+ 0.010489091277122498
+ ]
+ },
+ {
+ "state": "triage/4",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9516403675079346,
+ 0.04835968092083931
+ ],
+ "p_stock_bf16": [
+ 0.949153482913971,
+ 0.05084657296538353
+ ],
+ "p_fast": [
+ 0.9498092532157898,
+ 0.05019078403711319
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.03645458072423935,
+ 0.0668901652097702,
+ 0.12994635105133057,
+ 0.20574554800987244,
+ 0.08423464745283127,
+ 0.4767287075519562
+ ],
+ "p_stock_bf16": [
+ 0.03705758601427078,
+ 0.06606228649616241,
+ 0.12833712995052338,
+ 0.2036849856376648,
+ 0.08616146445274353,
+ 0.4786965250968933
+ ],
+ "p_fast": [
+ 0.03750967979431152,
+ 0.06686823815107346,
+ 0.12889191508293152,
+ 0.20396704971790314,
+ 0.08387155830860138,
+ 0.4788915514945984
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3825701177120209,
+ 0.6174299120903015
+ ],
+ "p_stock_bf16": [
+ 0.3826603591442108,
+ 0.6173396110534668
+ ],
+ "p_fast": [
+ 0.3800109028816223,
+ 0.6199891567230225
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.30471915006637573,
+ 0.2209567427635193,
+ 0.34811830520629883,
+ 0.12620577216148376
+ ],
+ "p_stock_bf16": [
+ 0.30808743834495544,
+ 0.22104191780090332,
+ 0.3463920056819916,
+ 0.12447857111692429
+ ],
+ "p_fast": [
+ 0.31200769543647766,
+ 0.2225467413663864,
+ 0.3413374423980713,
+ 0.12410808354616165
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.914557933807373,
+ 0.08544203639030457
+ ],
+ "p_stock_bf16": [
+ 0.9141375422477722,
+ 0.08586245030164719
+ ],
+ "p_fast": [
+ 0.915808916091919,
+ 0.08419107645750046
+ ]
+ },
+ {
+ "state": "triage/5",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9264069199562073,
+ 0.07359310984611511
+ ],
+ "p_stock_bf16": [
+ 0.9269675612449646,
+ 0.07303237915039062
+ ],
+ "p_fast": [
+ 0.929568350315094,
+ 0.07043161243200302
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.039129696786403656,
+ 0.19713838398456573,
+ 0.025492748245596886,
+ 0.034642286598682404,
+ 0.16664962470531464,
+ 0.5369472503662109
+ ],
+ "p_stock_bf16": [
+ 0.03975595906376839,
+ 0.1896650642156601,
+ 0.025668375194072723,
+ 0.03374044597148895,
+ 0.17506925761699677,
+ 0.5361008644104004
+ ],
+ "p_fast": [
+ 0.040243960916996,
+ 0.1883716732263565,
+ 0.02578124590218067,
+ 0.03310377895832062,
+ 0.1719336062669754,
+ 0.5405657291412354
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7484206557273865,
+ 0.25157931447029114
+ ],
+ "p_stock_bf16": [
+ 0.7434417009353638,
+ 0.2565583288669586
+ ],
+ "p_fast": [
+ 0.7498836517333984,
+ 0.2501163184642792
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.05583375692367554,
+ 0.09954751282930374,
+ 0.4533383846282959,
+ 0.3912803530693054
+ ],
+ "p_stock_bf16": [
+ 0.055868182331323624,
+ 0.0991106852889061,
+ 0.45873093605041504,
+ 0.38629019260406494
+ ],
+ "p_fast": [
+ 0.05326283350586891,
+ 0.09663509577512741,
+ 0.45653998851776123,
+ 0.39356204867362976
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6007707715034485,
+ 0.3992292284965515
+ ],
+ "p_stock_bf16": [
+ 0.5952572822570801,
+ 0.4047427177429199
+ ],
+ "p_fast": [
+ 0.6234341859817505,
+ 0.3765658140182495
+ ]
+ },
+ {
+ "state": "triage/6",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8552542924880981,
+ 0.1447456330060959
+ ],
+ "p_stock_bf16": [
+ 0.8586889505386353,
+ 0.14131103456020355
+ ],
+ "p_fast": [
+ 0.8487215638160706,
+ 0.15127839148044586
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.01829460635781288,
+ 0.04202770069241524,
+ 0.2798268795013428,
+ 0.2802577018737793,
+ 0.010803116485476494,
+ 0.36879006028175354
+ ],
+ "p_stock_bf16": [
+ 0.018012555316090584,
+ 0.04188045114278793,
+ 0.2806652784347534,
+ 0.2784811556339264,
+ 0.010589035227894783,
+ 0.3703715205192566
+ ],
+ "p_fast": [
+ 0.018939321860671043,
+ 0.04234827682375908,
+ 0.28602635860443115,
+ 0.27614542841911316,
+ 0.010707320645451546,
+ 0.3658333122730255
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9810092449188232,
+ 0.01899082027375698
+ ],
+ "p_stock_bf16": [
+ 0.9810219407081604,
+ 0.01897803694009781
+ ],
+ "p_fast": [
+ 0.9805806875228882,
+ 0.019419347867369652
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.4946100413799286,
+ 0.37923893332481384,
+ 0.10838776081800461,
+ 0.017763283103704453
+ ],
+ "p_stock_bf16": [
+ 0.4793633222579956,
+ 0.3851792514324188,
+ 0.1173010766506195,
+ 0.018156366422772408
+ ],
+ "p_fast": [
+ 0.48807770013809204,
+ 0.380115270614624,
+ 0.11346452683210373,
+ 0.01834256947040558
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9998832941055298,
+ 0.00011671962420223281
+ ],
+ "p_stock_bf16": [
+ 0.9998511075973511,
+ 0.00014883848780300468
+ ],
+ "p_fast": [
+ 0.9990226030349731,
+ 0.0009773502824828029
+ ]
+ },
+ {
+ "state": "triage/7",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9885086417198181,
+ 0.011491389945149422
+ ],
+ "p_stock_bf16": [
+ 0.9884026646614075,
+ 0.011597362346947193
+ ],
+ "p_fast": [
+ 0.9886244535446167,
+ 0.011375601403415203
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.11449223011732101,
+ 0.2214956432580948,
+ 0.2832673192024231,
+ 0.022097542881965637,
+ 0.015251493081450462,
+ 0.3433957099914551
+ ],
+ "p_stock_bf16": [
+ 0.11342591792345047,
+ 0.22272872924804688,
+ 0.27854713797569275,
+ 0.0226423442363739,
+ 0.01532057672739029,
+ 0.3473352789878845
+ ],
+ "p_fast": [
+ 0.11608028411865234,
+ 0.22550548613071442,
+ 0.28064611554145813,
+ 0.02193901129066944,
+ 0.015315896831452847,
+ 0.3405131995677948
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5090459585189819,
+ 0.49095404148101807
+ ],
+ "p_stock_bf16": [
+ 0.5117166042327881,
+ 0.4882833957672119
+ ],
+ "p_fast": [
+ 0.5114725828170776,
+ 0.48852741718292236
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.015272198244929314,
+ 0.1495506316423416,
+ 0.5121871829032898,
+ 0.3229900002479553
+ ],
+ "p_stock_bf16": [
+ 0.015496114268898964,
+ 0.1516900509595871,
+ 0.5091673731803894,
+ 0.3236464560031891
+ ],
+ "p_fast": [
+ 0.015178289264440536,
+ 0.14974422752857208,
+ 0.5105512738227844,
+ 0.3245261311531067
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9280251860618591,
+ 0.07197485864162445
+ ],
+ "p_stock_bf16": [
+ 0.930962085723877,
+ 0.06903792172670364
+ ],
+ "p_fast": [
+ 0.9263036847114563,
+ 0.07369626313447952
+ ]
+ },
+ {
+ "state": "triage/8",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9008024334907532,
+ 0.0991976410150528
+ ],
+ "p_stock_bf16": [
+ 0.8944211602210999,
+ 0.10557884722948074
+ ],
+ "p_fast": [
+ 0.9004859924316406,
+ 0.0995139554142952
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.06780702620744705,
+ 0.08060313016176224,
+ 0.16539226472377777,
+ 0.18834419548511505,
+ 0.028704779222607613,
+ 0.4691486060619354
+ ],
+ "p_stock_bf16": [
+ 0.066875159740448,
+ 0.07941621541976929,
+ 0.16199949383735657,
+ 0.1860964596271515,
+ 0.028538815677165985,
+ 0.4770738482475281
+ ],
+ "p_fast": [
+ 0.06618534028530121,
+ 0.07798539102077484,
+ 0.1615859419107437,
+ 0.19058077037334442,
+ 0.02780654840171337,
+ 0.4758559763431549
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.933054506778717,
+ 0.06694552302360535
+ ],
+ "p_stock_bf16": [
+ 0.9244152307510376,
+ 0.0755847916007042
+ ],
+ "p_fast": [
+ 0.9312127232551575,
+ 0.06878729164600372
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.336121529340744,
+ 0.26378756761550903,
+ 0.28905773162841797,
+ 0.11103314161300659
+ ],
+ "p_stock_bf16": [
+ 0.3312350809574127,
+ 0.2640836238861084,
+ 0.29231393337249756,
+ 0.11236737668514252
+ ],
+ "p_fast": [
+ 0.3304532766342163,
+ 0.2624332010746002,
+ 0.2939112186431885,
+ 0.11320226639509201
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9875450730323792,
+ 0.012454889714717865
+ ],
+ "p_stock_bf16": [
+ 0.9888850450515747,
+ 0.011115013621747494
+ ],
+ "p_fast": [
+ 0.9872281551361084,
+ 0.012771868146955967
+ ]
+ },
+ {
+ "state": "triage/9",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9731279611587524,
+ 0.02687203325331211
+ ],
+ "p_stock_bf16": [
+ 0.9728307723999023,
+ 0.027169233188033104
+ ],
+ "p_fast": [
+ 0.972727358341217,
+ 0.027272667735815048
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.007776996120810509,
+ 0.8459128141403198,
+ 0.009180194698274136,
+ 0.018750963732600212,
+ 0.009265405125916004,
+ 0.10911355167627335
+ ],
+ "p_stock_bf16": [
+ 0.007735501974821091,
+ 0.8465191721916199,
+ 0.009186135604977608,
+ 0.01826881431043148,
+ 0.009186135604977608,
+ 0.10910434275865555
+ ],
+ "p_fast": [
+ 0.0078092035837471485,
+ 0.8479341864585876,
+ 0.009129884652793407,
+ 0.018299352377653122,
+ 0.009129884652793407,
+ 0.1076975166797638
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.838813066482544,
+ 0.16118696331977844
+ ],
+ "p_stock_bf16": [
+ 0.84065181016922,
+ 0.15934816002845764
+ ],
+ "p_fast": [
+ 0.837619960308075,
+ 0.16238002479076385
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.07716948539018631,
+ 0.32050901651382446,
+ 0.4018648862838745,
+ 0.20045654475688934
+ ],
+ "p_stock_bf16": [
+ 0.0775127112865448,
+ 0.3200286626815796,
+ 0.4014059007167816,
+ 0.20105265080928802
+ ],
+ "p_fast": [
+ 0.07693102955818176,
+ 0.3188702464103699,
+ 0.4030897915363312,
+ 0.20110896229743958
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9491718411445618,
+ 0.05082815885543823
+ ],
+ "p_stock_bf16": [
+ 0.9491063356399536,
+ 0.05089372396469116
+ ],
+ "p_fast": [
+ 0.9531008005142212,
+ 0.046899259090423584
+ ]
+ },
+ {
+ "state": "triage/10",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9675112366676331,
+ 0.032488688826560974
+ ],
+ "p_stock_bf16": [
+ 0.968020498752594,
+ 0.03197956085205078
+ ],
+ "p_fast": [
+ 0.967777669429779,
+ 0.03222229704260826
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "intent",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.02766232006251812,
+ 0.07833411544561386,
+ 0.4743930995464325,
+ 0.03598523139953613,
+ 0.014720987528562546,
+ 0.3689042031764984
+ ],
+ "p_stock_bf16": [
+ 0.02751155197620392,
+ 0.07806757092475891,
+ 0.47450029850006104,
+ 0.03588182479143143,
+ 0.014497569762170315,
+ 0.3695411682128906
+ ],
+ "p_fast": [
+ 0.02726280502974987,
+ 0.07706011831760406,
+ 0.4776148200035095,
+ 0.03583628311753273,
+ 0.0145927295088768,
+ 0.36763325333595276
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "is_urgent",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9626188278198242,
+ 0.03738115355372429
+ ],
+ "p_stock_bf16": [
+ 0.9622496962547302,
+ 0.03775027021765709
+ ],
+ "p_fast": [
+ 0.9619649052619934,
+ 0.03803509101271629
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "frustration",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5732780694961548,
+ 0.26211413741111755,
+ 0.1290503442287445,
+ 0.03555748984217644
+ ],
+ "p_stock_bf16": [
+ 0.5735687613487244,
+ 0.2625989317893982,
+ 0.12829333543777466,
+ 0.03553890064358711
+ ],
+ "p_fast": [
+ 0.569918692111969,
+ 0.2640035152435303,
+ 0.12992769479751587,
+ 0.03615015372633934
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "refund_requested",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.996161699295044,
+ 0.0038383081555366516
+ ],
+ "p_stock_bf16": [
+ 0.9960082769393921,
+ 0.0039917281828820705
+ ],
+ "p_fast": [
+ 0.9959927201271057,
+ 0.004007289186120033
+ ]
+ },
+ {
+ "state": "triage/11",
+ "question": "churn_risk",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9891064763069153,
+ 0.01089353859424591
+ ],
+ "p_stock_bf16": [
+ 0.988447368144989,
+ 0.01155267097055912
+ ],
+ "p_fast": [
+ 0.9889705181121826,
+ 0.011029469780623913
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9249034523963928,
+ 0.0750965103507042
+ ],
+ "p_stock_bf16": [
+ 0.9245515465736389,
+ 0.0754484310746193
+ ],
+ "p_fast": [
+ 0.9244152307510376,
+ 0.0755847916007042
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9139643311500549,
+ 0.0860356017947197
+ ],
+ "p_stock_bf16": [
+ 0.9142141938209534,
+ 0.08578583598136902
+ ],
+ "p_fast": [
+ 0.9142141938209534,
+ 0.08578583598136902
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6928279399871826,
+ 0.307172030210495
+ ],
+ "p_stock_bf16": [
+ 0.6929017901420593,
+ 0.3070982098579407
+ ],
+ "p_fast": [
+ 0.6935248970985413,
+ 0.3064751625061035
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8171003460884094,
+ 0.1828996241092682
+ ],
+ "p_stock_bf16": [
+ 0.8135718703269958,
+ 0.18642809987068176
+ ],
+ "p_fast": [
+ 0.8134052157402039,
+ 0.18659478425979614
+ ]
+ },
+ {
+ "state": "moderation/0",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.12013990432024002,
+ 0.15027251839637756,
+ 0.3511520326137543,
+ 0.37843558192253113
+ ],
+ "p_stock_bf16": [
+ 0.12290246039628983,
+ 0.15054647624492645,
+ 0.3490923047065735,
+ 0.37745875120162964
+ ],
+ "p_fast": [
+ 0.12075484544038773,
+ 0.14905773103237152,
+ 0.34941574931144714,
+ 0.3807717263698578
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.955962061882019,
+ 0.04403799772262573
+ ],
+ "p_stock_bf16": [
+ 0.9553191661834717,
+ 0.044680871069431305
+ ],
+ "p_fast": [
+ 0.9551520943641663,
+ 0.04484790563583374
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9811826348304749,
+ 0.01881740801036358
+ ],
+ "p_stock_bf16": [
+ 0.9810945987701416,
+ 0.01890544965863228
+ ],
+ "p_fast": [
+ 0.9810945987701416,
+ 0.01890544965863228
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8559523820877075,
+ 0.14404763281345367
+ ],
+ "p_stock_bf16": [
+ 0.8589553833007812,
+ 0.14104463160037994
+ ],
+ "p_fast": [
+ 0.8554893136024475,
+ 0.1445106565952301
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8014840483665466,
+ 0.19851592183113098
+ ],
+ "p_stock_bf16": [
+ 0.7942951917648315,
+ 0.20570477843284607
+ ],
+ "p_fast": [
+ 0.7953701019287109,
+ 0.2046298384666443
+ ]
+ },
+ {
+ "state": "moderation/1",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2570229768753052,
+ 0.1490139216184616,
+ 0.3076266944408417,
+ 0.28633642196655273
+ ],
+ "p_stock_bf16": [
+ 0.260324090719223,
+ 0.1509585827589035,
+ 0.30613821744918823,
+ 0.28257912397384644
+ ],
+ "p_fast": [
+ 0.25960421562194824,
+ 0.14791801571846008,
+ 0.3017348647117615,
+ 0.2907429039478302
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9851654767990112,
+ 0.014834562316536903
+ ],
+ "p_stock_bf16": [
+ 0.9851576089859009,
+ 0.01484241709113121
+ ],
+ "p_fast": [
+ 0.9849852323532104,
+ 0.01501474715769291
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9851340055465698,
+ 0.014866044744849205
+ ],
+ "p_stock_bf16": [
+ 0.9851576089859009,
+ 0.01484241709113121
+ ],
+ "p_fast": [
+ 0.9854404330253601,
+ 0.014559518545866013
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9777073860168457,
+ 0.02229265309870243
+ ],
+ "p_stock_bf16": [
+ 0.9793540239334106,
+ 0.020645931363105774
+ ],
+ "p_fast": [
+ 0.9777987599372864,
+ 0.02220122702419758
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9317946434020996,
+ 0.06820531189441681
+ ],
+ "p_stock_bf16": [
+ 0.9328213930130005,
+ 0.06717856973409653
+ ],
+ "p_fast": [
+ 0.9326989054679871,
+ 0.06730107218027115
+ ]
+ },
+ {
+ "state": "moderation/2",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.6343872547149658,
+ 0.29902541637420654,
+ 0.045897550880908966,
+ 0.020689744502305984
+ ],
+ "p_stock_bf16": [
+ 0.6317017078399658,
+ 0.3019121587276459,
+ 0.045760318636894226,
+ 0.02062579058110714
+ ],
+ "p_fast": [
+ 0.6366196870803833,
+ 0.29837766289711,
+ 0.04469772055745125,
+ 0.020304853096604347
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9825068116188049,
+ 0.01749311573803425
+ ],
+ "p_stock_bf16": [
+ 0.9824230670928955,
+ 0.017576880753040314
+ ],
+ "p_fast": [
+ 0.982557475566864,
+ 0.017442483454942703
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9939368367195129,
+ 0.006063186097890139
+ ],
+ "p_stock_bf16": [
+ 0.9936856627464294,
+ 0.006314365193247795
+ ],
+ "p_fast": [
+ 0.9939024448394775,
+ 0.006097561679780483
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9386484026908875,
+ 0.06135157495737076
+ ],
+ "p_stock_bf16": [
+ 0.9393594861030579,
+ 0.06064055860042572
+ ],
+ "p_fast": [
+ 0.9383503198623657,
+ 0.0616496242582798
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6302990317344666,
+ 0.3697010278701782
+ ],
+ "p_stock_bf16": [
+ 0.6301160454750061,
+ 0.3698839247226715
+ ],
+ "p_fast": [
+ 0.6273806691169739,
+ 0.3726193606853485
+ ]
+ },
+ {
+ "state": "moderation/3",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.3822575509548187,
+ 0.243504598736763,
+ 0.23485027253627777,
+ 0.1393875777721405
+ ],
+ "p_stock_bf16": [
+ 0.3833010196685791,
+ 0.2445945292711258,
+ 0.23476530611515045,
+ 0.13733915984630585
+ ],
+ "p_fast": [
+ 0.3855634331703186,
+ 0.24175116419792175,
+ 0.23707525432109833,
+ 0.13561010360717773
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.18079549074172974,
+ 0.8192045092582703
+ ],
+ "p_stock_bf16": [
+ 0.18010665476322174,
+ 0.8198933005332947
+ ],
+ "p_fast": [
+ 0.18053968250751495,
+ 0.8194603323936462
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.18861573934555054,
+ 0.8113842606544495
+ ],
+ "p_stock_bf16": [
+ 0.1881747543811798,
+ 0.8118252158164978
+ ],
+ "p_fast": [
+ 0.1881747543811798,
+ 0.8118252158164978
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6191400289535522,
+ 0.38085994124412537
+ ],
+ "p_stock_bf16": [
+ 0.6185500621795654,
+ 0.38144993782043457
+ ],
+ "p_fast": [
+ 0.6167050004005432,
+ 0.3832949697971344
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9660330414772034,
+ 0.03396691754460335
+ ],
+ "p_stock_bf16": [
+ 0.9672868847846985,
+ 0.03271312639117241
+ ],
+ "p_fast": [
+ 0.9672868847846985,
+ 0.03271312639117241
+ ]
+ },
+ {
+ "state": "moderation/4",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.08258534967899323,
+ 0.2609964907169342,
+ 0.5500252842903137,
+ 0.10639287531375885
+ ],
+ "p_stock_bf16": [
+ 0.08272568136453629,
+ 0.25882577896118164,
+ 0.5500786900520325,
+ 0.10836982727050781
+ ],
+ "p_fast": [
+ 0.08224324136972427,
+ 0.2588284909725189,
+ 0.5522374510765076,
+ 0.10669083148241043
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9826089143753052,
+ 0.017391040921211243
+ ],
+ "p_stock_bf16": [
+ 0.982557475566864,
+ 0.017442483454942703
+ ],
+ "p_fast": [
+ 0.982624351978302,
+ 0.017375662922859192
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9797461628913879,
+ 0.020253824070096016
+ ],
+ "p_stock_bf16": [
+ 0.9796676635742188,
+ 0.020332353189587593
+ ],
+ "p_fast": [
+ 0.9796676635742188,
+ 0.020332353189587593
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9683158993721008,
+ 0.03168412670493126
+ ],
+ "p_stock_bf16": [
+ 0.9692078232765198,
+ 0.03079216368496418
+ ],
+ "p_fast": [
+ 0.968020498752594,
+ 0.03197956085205078
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.16894322633743286,
+ 0.8310567736625671
+ ],
+ "p_stock_bf16": [
+ 0.1710611879825592,
+ 0.8289388418197632
+ ],
+ "p_fast": [
+ 0.17217184603214264,
+ 0.8278281688690186
+ ]
+ },
+ {
+ "state": "moderation/5",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.27004456520080566,
+ 0.23373167216777802,
+ 0.34025830030441284,
+ 0.15596546232700348
+ ],
+ "p_stock_bf16": [
+ 0.27050602436065674,
+ 0.23456116020679474,
+ 0.339290589094162,
+ 0.15564224123954773
+ ],
+ "p_fast": [
+ 0.27216699719429016,
+ 0.23485185205936432,
+ 0.33805638551712036,
+ 0.15492470562458038
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9679447412490845,
+ 0.03205527737736702
+ ],
+ "p_stock_bf16": [
+ 0.967777669429779,
+ 0.03222229704260826
+ ],
+ "p_fast": [
+ 0.9684410095214844,
+ 0.03155902028083801
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9843999147415161,
+ 0.015600092709064484
+ ],
+ "p_stock_bf16": [
+ 0.9841545820236206,
+ 0.01584536023437977
+ ],
+ "p_fast": [
+ 0.9842759370803833,
+ 0.01572398841381073
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9328323602676392,
+ 0.06716759502887726
+ ],
+ "p_stock_bf16": [
+ 0.9339146018028259,
+ 0.06608538329601288
+ ],
+ "p_fast": [
+ 0.9303317666053772,
+ 0.0696682333946228
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8932691812515259,
+ 0.10673077404499054
+ ],
+ "p_stock_bf16": [
+ 0.8866904973983765,
+ 0.11330949515104294
+ ],
+ "p_fast": [
+ 0.8852320909500122,
+ 0.11476787179708481
+ ]
+ },
+ {
+ "state": "moderation/6",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2749302089214325,
+ 0.17856481671333313,
+ 0.29601383209228516,
+ 0.2504911422729492
+ ],
+ "p_stock_bf16": [
+ 0.27327316999435425,
+ 0.1797427386045456,
+ 0.2954787611961365,
+ 0.2515053153038025
+ ],
+ "p_fast": [
+ 0.2723390460014343,
+ 0.17860430479049683,
+ 0.2953326404094696,
+ 0.25372397899627686
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9986273050308228,
+ 0.0013727452605962753
+ ],
+ "p_stock_bf16": [
+ 0.9985674619674683,
+ 0.0014325366355478764
+ ],
+ "p_fast": [
+ 0.9986169338226318,
+ 0.001383117400109768
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9986945986747742,
+ 0.0013054306618869305
+ ],
+ "p_stock_bf16": [
+ 0.9987056255340576,
+ 0.0012943680630996823
+ ],
+ "p_fast": [
+ 0.9987106323242188,
+ 0.0012893282109871507
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9971616268157959,
+ 0.0028383766766637564
+ ],
+ "p_stock_bf16": [
+ 0.9972207546234131,
+ 0.0027791904285550117
+ ],
+ "p_fast": [
+ 0.9972207546234131,
+ 0.0027791904285550117
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9854265451431274,
+ 0.014573482796549797
+ ],
+ "p_stock_bf16": [
+ 0.9849274158477783,
+ 0.015072627924382687
+ ],
+ "p_fast": [
+ 0.9857181310653687,
+ 0.014281933195888996
+ ]
+ },
+ {
+ "state": "moderation/7",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5974613428115845,
+ 0.27034273743629456,
+ 0.10179945081472397,
+ 0.03039645589888096
+ ],
+ "p_stock_bf16": [
+ 0.5963622331619263,
+ 0.27197006344795227,
+ 0.10103404521942139,
+ 0.030633624643087387
+ ],
+ "p_fast": [
+ 0.5972955226898193,
+ 0.27027589082717896,
+ 0.10198582708835602,
+ 0.030442798510193825
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8529036045074463,
+ 0.14709636569023132
+ ],
+ "p_stock_bf16": [
+ 0.854763388633728,
+ 0.1452365517616272
+ ],
+ "p_fast": [
+ 0.8533025979995728,
+ 0.14669741690158844
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9688681960105896,
+ 0.03113178350031376
+ ],
+ "p_stock_bf16": [
+ 0.9685006141662598,
+ 0.031499382108449936
+ ],
+ "p_fast": [
+ 0.9687380790710449,
+ 0.03126191347837448
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.965154230594635,
+ 0.03484576568007469
+ ],
+ "p_stock_bf16": [
+ 0.9648551344871521,
+ 0.03514484688639641
+ ],
+ "p_fast": [
+ 0.9655115008354187,
+ 0.034488528966903687
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9680230617523193,
+ 0.03197691589593887
+ ],
+ "p_stock_bf16": [
+ 0.9690910577774048,
+ 0.03090895712375641
+ ],
+ "p_fast": [
+ 0.9694979190826416,
+ 0.03050205111503601
+ ]
+ },
+ {
+ "state": "moderation/8",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.15638655424118042,
+ 0.2918219566345215,
+ 0.45086419582366943,
+ 0.10092726349830627
+ ],
+ "p_stock_bf16": [
+ 0.15298150479793549,
+ 0.2907334864139557,
+ 0.45382851362228394,
+ 0.1024564802646637
+ ],
+ "p_fast": [
+ 0.1531909555196762,
+ 0.2941320836544037,
+ 0.4520154595375061,
+ 0.10066148638725281
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.990936815738678,
+ 0.00906322244554758
+ ],
+ "p_stock_bf16": [
+ 0.9908744096755981,
+ 0.00912563782185316
+ ],
+ "p_fast": [
+ 0.9911527037620544,
+ 0.008847355842590332
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9939631819725037,
+ 0.006036823615431786
+ ],
+ "p_stock_bf16": [
+ 0.9933071732521057,
+ 0.00669285049661994
+ ],
+ "p_fast": [
+ 0.9935117363929749,
+ 0.006488270126283169
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9746173024177551,
+ 0.025382665917277336
+ ],
+ "p_stock_bf16": [
+ 0.975669801235199,
+ 0.02433019131422043
+ ],
+ "p_fast": [
+ 0.975296139717102,
+ 0.024703867733478546
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6642186045646667,
+ 0.33578136563301086
+ ],
+ "p_stock_bf16": [
+ 0.670284628868103,
+ 0.329715371131897
+ ],
+ "p_fast": [
+ 0.6655192375183105,
+ 0.33448073267936707
+ ]
+ },
+ {
+ "state": "moderation/9",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.45076873898506165,
+ 0.3377600908279419,
+ 0.16410930454730988,
+ 0.047361861914396286
+ ],
+ "p_stock_bf16": [
+ 0.4510742127895355,
+ 0.34048864245414734,
+ 0.16089436411857605,
+ 0.04754287004470825
+ ],
+ "p_fast": [
+ 0.4527100622653961,
+ 0.33774226903915405,
+ 0.16145814955234528,
+ 0.04808952286839485
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9556035399436951,
+ 0.04439644142985344
+ ],
+ "p_stock_bf16": [
+ 0.9554855227470398,
+ 0.044514428824186325
+ ],
+ "p_fast": [
+ 0.9556514620780945,
+ 0.0443485826253891
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9755018949508667,
+ 0.02449815534055233
+ ],
+ "p_stock_bf16": [
+ 0.975296139717102,
+ 0.024703867733478546
+ ],
+ "p_fast": [
+ 0.975296139717102,
+ 0.024703867733478546
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9696269035339355,
+ 0.030373062938451767
+ ],
+ "p_stock_bf16": [
+ 0.9692078232765198,
+ 0.03079216368496418
+ ],
+ "p_fast": [
+ 0.9696707129478455,
+ 0.030329255387187004
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9871689081192017,
+ 0.012831129133701324
+ ],
+ "p_stock_bf16": [
+ 0.9870296716690063,
+ 0.012970385141670704
+ ],
+ "p_fast": [
+ 0.9869794845581055,
+ 0.013020490296185017
+ ]
+ },
+ {
+ "state": "moderation/10",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.3204658329486847,
+ 0.3182946741580963,
+ 0.20673127472400665,
+ 0.15450820326805115
+ ],
+ "p_stock_bf16": [
+ 0.31928426027297974,
+ 0.31866127252578735,
+ 0.20735685527324677,
+ 0.15469762682914734
+ ],
+ "p_fast": [
+ 0.318727046251297,
+ 0.318727046251297,
+ 0.20872032642364502,
+ 0.15382559597492218
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "toxic",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9834228754043579,
+ 0.016577094793319702
+ ],
+ "p_stock_bf16": [
+ 0.9836598634719849,
+ 0.01634012721478939
+ ],
+ "p_fast": [
+ 0.9834704995155334,
+ 0.01652955822646618
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "harassment",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.996426522731781,
+ 0.0035734723787754774
+ ],
+ "p_stock_bf16": [
+ 0.9965837001800537,
+ 0.00341627886518836
+ ],
+ "p_fast": [
+ 0.9962638020515442,
+ 0.0037362207658588886
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "threat",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9983080625534058,
+ 0.0016919723711907864
+ ],
+ "p_stock_bf16": [
+ 0.9984269142150879,
+ 0.0015731120947748423
+ ],
+ "p_fast": [
+ 0.9983189702033997,
+ 0.001680941553786397
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9977206587791443,
+ 0.002279271837323904
+ ],
+ "p_stock_bf16": [
+ 0.9977127313613892,
+ 0.0022872271947562695
+ ],
+ "p_fast": [
+ 0.9977216124534607,
+ 0.002278330735862255
+ ]
+ },
+ {
+ "state": "moderation/11",
+ "question": "severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.6300764679908752,
+ 0.276387482881546,
+ 0.06863974034786224,
+ 0.024896353483200073
+ ],
+ "p_stock_bf16": [
+ 0.6285822987556458,
+ 0.27784425020217896,
+ 0.06862268596887589,
+ 0.02495076321065426
+ ],
+ "p_fast": [
+ 0.6285355091094971,
+ 0.2778235673904419,
+ 0.06888613849878311,
+ 0.024754753336310387
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9848158359527588,
+ 0.01518422830849886
+ ],
+ "p_stock_bf16": [
+ 0.9845753908157349,
+ 0.01542455144226551
+ ],
+ "p_fast": [
+ 0.9842759370803833,
+ 0.01572398841381073
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8343393206596375,
+ 0.16566069424152374
+ ],
+ "p_stock_bf16": [
+ 0.8420855402946472,
+ 0.15791445970535278
+ ],
+ "p_fast": [
+ 0.8433797955513,
+ 0.15662017464637756
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7587581276893616,
+ 0.24124185740947723
+ ],
+ "p_stock_bf16": [
+ 0.7618929743766785,
+ 0.23810699582099915
+ ],
+ "p_fast": [
+ 0.7611836194992065,
+ 0.23881636559963226
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.03488408774137497,
+ 0.11611437797546387,
+ 0.387145459651947,
+ 0.46185603737831116
+ ],
+ "p_stock_bf16": [
+ 0.034381672739982605,
+ 0.11518137902021408,
+ 0.3904153108596802,
+ 0.4600217044353485
+ ],
+ "p_fast": [
+ 0.03471510857343674,
+ 0.11573193222284317,
+ 0.38506993651390076,
+ 0.46448302268981934
+ ]
+ },
+ {
+ "state": "guard/0",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.3391225039958954,
+ 0.04174385592341423,
+ 0.0269475094974041,
+ 0.42629510164260864,
+ 0.032247353345155716,
+ 0.13364359736442566
+ ],
+ "p_stock_bf16": [
+ 0.3405338525772095,
+ 0.042789630591869354,
+ 0.02698708325624466,
+ 0.42545998096466064,
+ 0.032299309968948364,
+ 0.1319301426410675
+ ],
+ "p_fast": [
+ 0.33793148398399353,
+ 0.04296316206455231,
+ 0.027309054508805275,
+ 0.42718544602394104,
+ 0.03217792510986328,
+ 0.13243283331394196
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9927472472190857,
+ 0.007252735085785389
+ ],
+ "p_stock_bf16": [
+ 0.9927951693534851,
+ 0.007204768713563681
+ ],
+ "p_fast": [
+ 0.9925103187561035,
+ 0.0074896239675581455
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8958684206008911,
+ 0.10413163155317307
+ ],
+ "p_stock_bf16": [
+ 0.895339846611023,
+ 0.10466020554304123
+ ],
+ "p_fast": [
+ 0.8975157737731934,
+ 0.10248422622680664
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6366130113601685,
+ 0.36338695883750916
+ ],
+ "p_stock_bf16": [
+ 0.6389479041099548,
+ 0.3610520660877228
+ ],
+ "p_fast": [
+ 0.6362400650978088,
+ 0.3637598752975464
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0338304229080677,
+ 0.06095995754003525,
+ 0.30135539174079895,
+ 0.603854238986969
+ ],
+ "p_stock_bf16": [
+ 0.03367773815989494,
+ 0.05980299785733223,
+ 0.30016642808914185,
+ 0.6063528656959534
+ ],
+ "p_fast": [
+ 0.033725716173648834,
+ 0.06083130091428757,
+ 0.3029516339302063,
+ 0.6024913191795349
+ ]
+ },
+ {
+ "state": "guard/1",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.2601044774055481,
+ 0.31739547848701477,
+ 0.01272143330425024,
+ 0.028285974636673927,
+ 0.3531803786754608,
+ 0.028312217444181442
+ ],
+ "p_stock_bf16": [
+ 0.25975796580314636,
+ 0.31826215982437134,
+ 0.01253469567745924,
+ 0.028468921780586243,
+ 0.35228410363197327,
+ 0.028692204505205154
+ ],
+ "p_fast": [
+ 0.26471811532974243,
+ 0.30948683619499207,
+ 0.012674639001488686,
+ 0.028562741354107857,
+ 0.3562171757221222,
+ 0.02834046259522438
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9817904233932495,
+ 0.01820952072739601
+ ],
+ "p_stock_bf16": [
+ 0.982624351978302,
+ 0.017375662922859192
+ ],
+ "p_fast": [
+ 0.982624351978302,
+ 0.017375662922859192
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9153319597244263,
+ 0.08466808497905731
+ ],
+ "p_stock_bf16": [
+ 0.9171544313430786,
+ 0.08284562826156616
+ ],
+ "p_fast": [
+ 0.917302668094635,
+ 0.08269733935594559
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9010818600654602,
+ 0.09891817718744278
+ ],
+ "p_stock_bf16": [
+ 0.9027811288833618,
+ 0.0972188413143158
+ ],
+ "p_fast": [
+ 0.9038048982620239,
+ 0.09619517624378204
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.5014544129371643,
+ 0.3993522524833679,
+ 0.07377302646636963,
+ 0.025420313701033592
+ ],
+ "p_stock_bf16": [
+ 0.49975043535232544,
+ 0.4015607535839081,
+ 0.07341586798429489,
+ 0.025272928178310394
+ ],
+ "p_fast": [
+ 0.5058509707450867,
+ 0.3970469832420349,
+ 0.07230763137340546,
+ 0.024794381111860275
+ ]
+ },
+ {
+ "state": "guard/2",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.21425652503967285,
+ 0.03020607866346836,
+ 0.1317135989665985,
+ 0.24805848300457,
+ 0.03104798123240471,
+ 0.34471726417541504
+ ],
+ "p_stock_bf16": [
+ 0.20977245271205902,
+ 0.03022058494389057,
+ 0.13050530850887299,
+ 0.24911093711853027,
+ 0.03045760840177536,
+ 0.34993311762809753
+ ],
+ "p_fast": [
+ 0.21341930329799652,
+ 0.03061114251613617,
+ 0.13193398714065552,
+ 0.249634250998497,
+ 0.030851228162646294,
+ 0.34355008602142334
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.18380992114543915,
+ 0.8161900639533997
+ ],
+ "p_stock_bf16": [
+ 0.18257123231887817,
+ 0.8174288272857666
+ ],
+ "p_fast": [
+ 0.18257123231887817,
+ 0.8174288272857666
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.23025445640087128,
+ 0.7697455286979675
+ ],
+ "p_stock_bf16": [
+ 0.23213602602481842,
+ 0.767863929271698
+ ],
+ "p_fast": [
+ 0.22764098644256592,
+ 0.7723590135574341
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9547452926635742,
+ 0.04525475576519966
+ ],
+ "p_stock_bf16": [
+ 0.9552357196807861,
+ 0.04476431384682655
+ ],
+ "p_fast": [
+ 0.954562783241272,
+ 0.04543721675872803
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.20860610902309418,
+ 0.18134969472885132,
+ 0.29079294204711914,
+ 0.3192512094974518
+ ],
+ "p_stock_bf16": [
+ 0.20716634392738342,
+ 0.17963802814483643,
+ 0.2904445230960846,
+ 0.32275107502937317
+ ],
+ "p_fast": [
+ 0.20497488975524902,
+ 0.17983290553092957,
+ 0.29019227623939514,
+ 0.32499992847442627
+ ]
+ },
+ {
+ "state": "guard/3",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.15682639181613922,
+ 0.2380664348602295,
+ 0.10981667786836624,
+ 0.2613714933395386,
+ 0.18578195571899414,
+ 0.04813706874847412
+ ],
+ "p_stock_bf16": [
+ 0.15555955469608307,
+ 0.23812833428382874,
+ 0.11095646768808365,
+ 0.2594975531101227,
+ 0.18690907955169678,
+ 0.0489489808678627
+ ],
+ "p_fast": [
+ 0.1556766927242279,
+ 0.2373785823583603,
+ 0.1103912964463234,
+ 0.2627541422843933,
+ 0.18632058799266815,
+ 0.047478701919317245
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8540359735488892,
+ 0.14596398174762726
+ ],
+ "p_stock_bf16": [
+ 0.8649391531944275,
+ 0.1350608766078949
+ ],
+ "p_fast": [
+ 0.8504685759544373,
+ 0.14953136444091797
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8814943432807922,
+ 0.11850562691688538
+ ],
+ "p_stock_bf16": [
+ 0.8802834153175354,
+ 0.11971654742956161
+ ],
+ "p_fast": [
+ 0.8790948390960693,
+ 0.12090512365102768
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.967627763748169,
+ 0.03237223997712135
+ ],
+ "p_stock_bf16": [
+ 0.968020498752594,
+ 0.03197956085205078
+ ],
+ "p_fast": [
+ 0.9680808186531067,
+ 0.03191915154457092
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.06822484731674194,
+ 0.13737277686595917,
+ 0.39298364520072937,
+ 0.4014187157154083
+ ],
+ "p_stock_bf16": [
+ 0.06780196726322174,
+ 0.1363631784915924,
+ 0.3917004466056824,
+ 0.4041343927383423
+ ],
+ "p_fast": [
+ 0.06730536371469498,
+ 0.13649269938468933,
+ 0.39188113808631897,
+ 0.4043208062648773
+ ]
+ },
+ {
+ "state": "guard/4",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.11639373004436493,
+ 0.12373795360326767,
+ 0.17377381026744843,
+ 0.16929568350315094,
+ 0.07642018795013428,
+ 0.34037864208221436
+ ],
+ "p_stock_bf16": [
+ 0.11506938189268112,
+ 0.12249071151018143,
+ 0.1754598468542099,
+ 0.16612204909324646,
+ 0.07665273547172546,
+ 0.3442053496837616
+ ],
+ "p_fast": [
+ 0.11644638329744339,
+ 0.12014279514551163,
+ 0.17480671405792236,
+ 0.1681099832057953,
+ 0.07757001370191574,
+ 0.3429240882396698
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.08370641618967056,
+ 0.9162935614585876
+ ],
+ "p_stock_bf16": [
+ 0.08494709432125092,
+ 0.9150528311729431
+ ],
+ "p_fast": [
+ 0.08694165199995041,
+ 0.9130584001541138
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3785058557987213,
+ 0.6214941740036011
+ ],
+ "p_stock_bf16": [
+ 0.37897607684135437,
+ 0.6210238933563232
+ ],
+ "p_fast": [
+ 0.38496991991996765,
+ 0.61503005027771
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7780634164810181,
+ 0.22193655371665955
+ ],
+ "p_stock_bf16": [
+ 0.7743061780929565,
+ 0.22569382190704346
+ ],
+ "p_fast": [
+ 0.7810176014900208,
+ 0.21898238360881805
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.20972195267677307,
+ 0.24862968921661377,
+ 0.35670939087867737,
+ 0.1849389374256134
+ ],
+ "p_stock_bf16": [
+ 0.2099088877439499,
+ 0.2522113025188446,
+ 0.35290706157684326,
+ 0.18497277796268463
+ ],
+ "p_fast": [
+ 0.2131618857383728,
+ 0.24860380589962006,
+ 0.35402804613113403,
+ 0.18420620262622833
+ ]
+ },
+ {
+ "state": "guard/5",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.23745842278003693,
+ 0.09393908828496933,
+ 0.03363722562789917,
+ 0.22656786441802979,
+ 0.23654960095882416,
+ 0.17184776067733765
+ ],
+ "p_stock_bf16": [
+ 0.23964950442314148,
+ 0.09513998031616211,
+ 0.03339727595448494,
+ 0.2251298874616623,
+ 0.23616458475589752,
+ 0.17051878571510315
+ ],
+ "p_fast": [
+ 0.23432330787181854,
+ 0.09248203039169312,
+ 0.033494770526885986,
+ 0.2277802675962448,
+ 0.23964597284793854,
+ 0.17227371037006378
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3019067049026489,
+ 0.6980932950973511
+ ],
+ "p_stock_bf16": [
+ 0.29128316044807434,
+ 0.708716869354248
+ ],
+ "p_fast": [
+ 0.306215763092041,
+ 0.693784236907959
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.659529983997345,
+ 0.34046998620033264
+ ],
+ "p_stock_bf16": [
+ 0.6657366752624512,
+ 0.3342633843421936
+ ],
+ "p_fast": [
+ 0.6774740219116211,
+ 0.3225259780883789
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.4156767725944519,
+ 0.5843231678009033
+ ],
+ "p_stock_bf16": [
+ 0.4031555652618408,
+ 0.5968444347381592
+ ],
+ "p_fast": [
+ 0.4026857018470764,
+ 0.5973142981529236
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.08883614838123322,
+ 0.13282839953899384,
+ 0.2743772268295288,
+ 0.5039582848548889
+ ],
+ "p_stock_bf16": [
+ 0.0850861594080925,
+ 0.1317196935415268,
+ 0.2730568051338196,
+ 0.5101373195648193
+ ],
+ "p_fast": [
+ 0.08472185581922531,
+ 0.13115571439266205,
+ 0.2761693298816681,
+ 0.5079531073570251
+ ]
+ },
+ {
+ "state": "guard/6",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.14637786149978638,
+ 0.3235582113265991,
+ 0.03658990561962128,
+ 0.04097359627485275,
+ 0.4122118353843689,
+ 0.04028867930173874
+ ],
+ "p_stock_bf16": [
+ 0.14300182461738586,
+ 0.3131081461906433,
+ 0.035682983696460724,
+ 0.04011944681406021,
+ 0.4279681444168091,
+ 0.04011944681406021
+ ],
+ "p_fast": [
+ 0.14403627812862396,
+ 0.3225073516368866,
+ 0.03604325279593468,
+ 0.04084235802292824,
+ 0.4157284200191498,
+ 0.04084235802292824
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7632602453231812,
+ 0.23673976957798004
+ ],
+ "p_stock_bf16": [
+ 0.7329742908477783,
+ 0.26702573895454407
+ ],
+ "p_fast": [
+ 0.7741994857788086,
+ 0.2258004993200302
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8913726210594177,
+ 0.10862734913825989
+ ],
+ "p_stock_bf16": [
+ 0.8954769968986511,
+ 0.10452301055192947
+ ],
+ "p_fast": [
+ 0.897335946559906,
+ 0.10266400873661041
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8998916149139404,
+ 0.10010836273431778
+ ],
+ "p_stock_bf16": [
+ 0.9010533690452576,
+ 0.09894658625125885
+ ],
+ "p_fast": [
+ 0.8998278379440308,
+ 0.10017222166061401
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.46722984313964844,
+ 0.369934618473053,
+ 0.11892163753509521,
+ 0.04391384497284889
+ ],
+ "p_stock_bf16": [
+ 0.4667341411113739,
+ 0.3692170977592468,
+ 0.11977947503328323,
+ 0.04426927864551544
+ ],
+ "p_fast": [
+ 0.46636468172073364,
+ 0.3689248263835907,
+ 0.12047620862722397,
+ 0.04423423856496811
+ ]
+ },
+ {
+ "state": "guard/7",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.19406408071517944,
+ 0.2018871158361435,
+ 0.26675018668174744,
+ 0.09229136258363724,
+ 0.042288582772016525,
+ 0.20271864533424377
+ ],
+ "p_stock_bf16": [
+ 0.192929208278656,
+ 0.20547237992286682,
+ 0.2664208710193634,
+ 0.09153467416763306,
+ 0.04273418337106705,
+ 0.20090872049331665
+ ],
+ "p_fast": [
+ 0.19217632710933685,
+ 0.2055719643831253,
+ 0.2678546905517578,
+ 0.09148965775966644,
+ 0.0428803376853466,
+ 0.20002701878547668
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9983421564102173,
+ 0.001657813205383718
+ ],
+ "p_stock_bf16": [
+ 0.9983834028244019,
+ 0.001616650028154254
+ ],
+ "p_fast": [
+ 0.9982725381851196,
+ 0.0017274586716666818
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9729209542274475,
+ 0.027079008519649506
+ ],
+ "p_stock_bf16": [
+ 0.9731387495994568,
+ 0.026861201971769333
+ ],
+ "p_fast": [
+ 0.973993182182312,
+ 0.026006784290075302
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5183061361312866,
+ 0.4816938638687134
+ ],
+ "p_stock_bf16": [
+ 0.517327070236206,
+ 0.48267295956611633
+ ],
+ "p_fast": [
+ 0.5193384885787964,
+ 0.4806615114212036
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.023643840104341507,
+ 0.15977902710437775,
+ 0.25296124815940857,
+ 0.5636158585548401
+ ],
+ "p_stock_bf16": [
+ 0.02337218075990677,
+ 0.15909716486930847,
+ 0.25127461552619934,
+ 0.566256046295166
+ ],
+ "p_fast": [
+ 0.022561175748705864,
+ 0.15238143503665924,
+ 0.2522178292274475,
+ 0.5728394985198975
+ ]
+ },
+ {
+ "state": "guard/8",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.28367963433265686,
+ 0.05565132573246956,
+ 0.05322139710187912,
+ 0.11245077848434448,
+ 0.22624066472053528,
+ 0.2687561810016632
+ ],
+ "p_stock_bf16": [
+ 0.28227752447128296,
+ 0.05690186843276024,
+ 0.05303836986422539,
+ 0.11272171139717102,
+ 0.23117700219154358,
+ 0.2638835608959198
+ ],
+ "p_fast": [
+ 0.2892058491706848,
+ 0.055195897817611694,
+ 0.05349769815802574,
+ 0.11193518340587616,
+ 0.2206617146730423,
+ 0.2695036828517914
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9274953007698059,
+ 0.07250470668077469
+ ],
+ "p_stock_bf16": [
+ 0.9235233068466187,
+ 0.07647661864757538
+ ],
+ "p_fast": [
+ 0.9269675612449646,
+ 0.07303237915039062
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9180071353912354,
+ 0.08199288696050644
+ ],
+ "p_stock_bf16": [
+ 0.9207183718681335,
+ 0.07928158342838287
+ ],
+ "p_fast": [
+ 0.9177460074424744,
+ 0.08225394040346146
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8884492516517639,
+ 0.11155081540346146
+ ],
+ "p_stock_bf16": [
+ 0.8887588381767273,
+ 0.11124120652675629
+ ],
+ "p_fast": [
+ 0.8888553380966187,
+ 0.11114469170570374
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2410876750946045,
+ 0.35780009627342224,
+ 0.19815438985824585,
+ 0.20295782387256622
+ ],
+ "p_stock_bf16": [
+ 0.23907342553138733,
+ 0.3574925363063812,
+ 0.2005351334810257,
+ 0.20289897918701172
+ ],
+ "p_fast": [
+ 0.23702497780323029,
+ 0.3614199459552765,
+ 0.19881689548492432,
+ 0.2027382105588913
+ ]
+ },
+ {
+ "state": "guard/9",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.08929187059402466,
+ 0.20355252921581268,
+ 0.1244087740778923,
+ 0.04064306989312172,
+ 0.1756647676229477,
+ 0.36643898487091064
+ ],
+ "p_stock_bf16": [
+ 0.08579842746257782,
+ 0.20326048135757446,
+ 0.1233815848827362,
+ 0.04068692773580551,
+ 0.17708149552345276,
+ 0.36979106068611145
+ ],
+ "p_fast": [
+ 0.08764826506376266,
+ 0.20242469012737274,
+ 0.12360386550426483,
+ 0.04060131683945656,
+ 0.1767088919878006,
+ 0.3690129816532135
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.984460175037384,
+ 0.015539758838713169
+ ],
+ "p_stock_bf16": [
+ 0.985042929649353,
+ 0.014957087114453316
+ ],
+ "p_fast": [
+ 0.9849852323532104,
+ 0.01501474715769291
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8587174415588379,
+ 0.14128261804580688
+ ],
+ "p_stock_bf16": [
+ 0.8576489090919495,
+ 0.14235107600688934
+ ],
+ "p_fast": [
+ 0.8630456924438477,
+ 0.13695430755615234
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6365188360214233,
+ 0.36348119378089905
+ ],
+ "p_stock_bf16": [
+ 0.6400524973869324,
+ 0.3599475026130676
+ ],
+ "p_fast": [
+ 0.638102650642395,
+ 0.361897349357605
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.04678545519709587,
+ 0.1299351155757904,
+ 0.2839876115322113,
+ 0.5392918586730957
+ ],
+ "p_stock_bf16": [
+ 0.045889418572187424,
+ 0.12895166873931885,
+ 0.28331151604652405,
+ 0.5418474674224854
+ ],
+ "p_fast": [
+ 0.046153366565704346,
+ 0.12818241119384766,
+ 0.28494107723236084,
+ 0.5407231450080872
+ ]
+ },
+ {
+ "state": "guard/10",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.24041950702667236,
+ 0.16486990451812744,
+ 0.04929640144109726,
+ 0.038499459624290466,
+ 0.40653443336486816,
+ 0.1003803089261055
+ ],
+ "p_stock_bf16": [
+ 0.24066005647182465,
+ 0.165726438164711,
+ 0.04946938529610634,
+ 0.03852679207921028,
+ 0.4046071469783783,
+ 0.10101017355918884
+ ],
+ "p_fast": [
+ 0.2389344871044159,
+ 0.16262124478816986,
+ 0.05008338764309883,
+ 0.03915764391422272,
+ 0.40803202986717224,
+ 0.10117122530937195
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "jailbreak",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9987101554870605,
+ 0.0012898908462375402
+ ],
+ "p_stock_bf16": [
+ 0.9987157583236694,
+ 0.0012843082658946514
+ ],
+ "p_fast": [
+ 0.9987207055091858,
+ 0.0012793075293302536
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "prompt_injection",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9770605564117432,
+ 0.022939473390579224
+ ],
+ "p_stock_bf16": [
+ 0.9774571061134338,
+ 0.022542964667081833
+ ],
+ "p_fast": [
+ 0.9764904975891113,
+ 0.023509450256824493
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "sensitive_data",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9678712487220764,
+ 0.0321287140250206
+ ],
+ "p_stock_bf16": [
+ 0.9678993225097656,
+ 0.032100703567266464
+ ],
+ "p_fast": [
+ 0.9685006141662598,
+ 0.031499382108449936
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "harm_severity",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.4592909812927246,
+ 0.36317652463912964,
+ 0.10236763209104538,
+ 0.07516488432884216
+ ],
+ "p_stock_bf16": [
+ 0.4591200351715088,
+ 0.3631938397884369,
+ 0.10244352370500565,
+ 0.07524263113737106
+ ],
+ "p_fast": [
+ 0.4514319598674774,
+ 0.3655807375907898,
+ 0.10515058040618896,
+ 0.07783663272857666
+ ]
+ },
+ {
+ "state": "guard/11",
+ "question": "topic",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.23899833858013153,
+ 0.13979384303092957,
+ 0.14536887407302856,
+ 0.07561320066452026,
+ 0.07931211590766907,
+ 0.3209136724472046
+ ],
+ "p_stock_bf16": [
+ 0.23472048342227936,
+ 0.1426435112953186,
+ 0.14774750173091888,
+ 0.07516784965991974,
+ 0.0812758281826973,
+ 0.31844478845596313
+ ],
+ "p_fast": [
+ 0.2328144758939743,
+ 0.14203894138336182,
+ 0.14712131023406982,
+ 0.07602797448635101,
+ 0.08030153810977936,
+ 0.3216957449913025
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.15363943576812744,
+ 0.5411243438720703,
+ 0.20558011531829834,
+ 0.09965616464614868
+ ],
+ "p_stock_bf16": [
+ 0.15378369390964508,
+ 0.543084979057312,
+ 0.20422828197479248,
+ 0.0989031121134758
+ ],
+ "p_fast": [
+ 0.15308180451393127,
+ 0.5437831878662109,
+ 0.20449085533618927,
+ 0.0986441895365715
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.14609380066394806,
+ 0.22429321706295013,
+ 0.24897204339504242,
+ 0.14454080164432526,
+ 0.06140107661485672,
+ 0.17469905316829681
+ ],
+ "p_stock_bf16": [
+ 0.14066918194293976,
+ 0.23192426562309265,
+ 0.2468820959329605,
+ 0.14400506019592285,
+ 0.06145380809903145,
+ 0.17506560683250427
+ ],
+ "p_fast": [
+ 0.14432792365550995,
+ 0.22529269754886627,
+ 0.245510071516037,
+ 0.14320476353168488,
+ 0.06063671037554741,
+ 0.18102775514125824
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9460921287536621,
+ 0.05390787124633789
+ ],
+ "p_stock_bf16": [
+ 0.9480584859848022,
+ 0.051941562443971634
+ ],
+ "p_fast": [
+ 0.9465966820716858,
+ 0.053403329104185104
+ ]
+ },
+ {
+ "state": "router/0",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5182129740715027,
+ 0.4817869961261749
+ ],
+ "p_stock_bf16": [
+ 0.514514148235321,
+ 0.48548582196235657
+ ],
+ "p_fast": [
+ 0.5146061182022095,
+ 0.48539385199546814
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.061001695692539215,
+ 0.2587133049964905,
+ 0.3934936225414276,
+ 0.2867914140224457
+ ],
+ "p_stock_bf16": [
+ 0.06104345992207527,
+ 0.2618156671524048,
+ 0.38959261775016785,
+ 0.2875482439994812
+ ],
+ "p_fast": [
+ 0.06186654418706894,
+ 0.2616150379180908,
+ 0.3910086154937744,
+ 0.28550985455513
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9106506705284119,
+ 0.01608418859541416,
+ 0.021707722917199135,
+ 0.017790555953979492,
+ 0.014671149663627148,
+ 0.01909572072327137
+ ],
+ "p_stock_bf16": [
+ 0.910634458065033,
+ 0.01603989489376545,
+ 0.021249407902359962,
+ 0.017893794924020767,
+ 0.014834477566182613,
+ 0.01934780739247799
+ ],
+ "p_fast": [
+ 0.9117276072502136,
+ 0.015934176743030548,
+ 0.021109355613589287,
+ 0.017500270158052444,
+ 0.014508233405649662,
+ 0.01922028698027134
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.47418609261512756,
+ 0.5258139371871948
+ ],
+ "p_stock_bf16": [
+ 0.47755417227745056,
+ 0.522445797920227
+ ],
+ "p_fast": [
+ 0.4676963984966278,
+ 0.5323035717010498
+ ]
+ },
+ {
+ "state": "router/1",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6362482905387878,
+ 0.36375170946121216
+ ],
+ "p_stock_bf16": [
+ 0.6308697462081909,
+ 0.36913031339645386
+ ],
+ "p_fast": [
+ 0.6299168467521667,
+ 0.3700830936431885
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.11406733840703964,
+ 0.6427571177482605,
+ 0.19504785537719727,
+ 0.048127733170986176
+ ],
+ "p_stock_bf16": [
+ 0.11357227712869644,
+ 0.642332136631012,
+ 0.19590024650096893,
+ 0.04819531366229057
+ ],
+ "p_fast": [
+ 0.11213021725416183,
+ 0.6433771252632141,
+ 0.19621895253658295,
+ 0.0482737235724926
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.06784193217754364,
+ 0.07607923448085785,
+ 0.1005159467458725,
+ 0.09539924561977386,
+ 0.05125369876623154,
+ 0.6089099049568176
+ ],
+ "p_stock_bf16": [
+ 0.06853372603654861,
+ 0.0758599266409874,
+ 0.10128628462553024,
+ 0.09294553101062775,
+ 0.05173197761178017,
+ 0.6096426248550415
+ ],
+ "p_fast": [
+ 0.06814374774694443,
+ 0.07542824745178223,
+ 0.09992619603872299,
+ 0.09314145892858505,
+ 0.050640132278203964,
+ 0.612720251083374
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9281124472618103,
+ 0.07188758999109268
+ ],
+ "p_stock_bf16": [
+ 0.9293122291564941,
+ 0.07068779319524765
+ ],
+ "p_fast": [
+ 0.9315870404243469,
+ 0.0684129074215889
+ ]
+ },
+ {
+ "state": "router/2",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.901957094669342,
+ 0.09804288297891617
+ ],
+ "p_stock_bf16": [
+ 0.9011839628219604,
+ 0.09881607443094254
+ ],
+ "p_fast": [
+ 0.9011839628219604,
+ 0.09881607443094254
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.2526102364063263,
+ 0.5600556135177612,
+ 0.15408360958099365,
+ 0.03325054049491882
+ ],
+ "p_stock_bf16": [
+ 0.25171929597854614,
+ 0.5628437399864197,
+ 0.15267546474933624,
+ 0.03276143595576286
+ ],
+ "p_fast": [
+ 0.2527602016925812,
+ 0.5651712417602539,
+ 0.1491716355085373,
+ 0.03289690986275673
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.29658329486846924,
+ 0.21880574524402618,
+ 0.18251341581344604,
+ 0.12181292474269867,
+ 0.09264285117387772,
+ 0.08764180541038513
+ ],
+ "p_stock_bf16": [
+ 0.2987246811389923,
+ 0.22417186200618744,
+ 0.17942489683628082,
+ 0.11859261244535446,
+ 0.09164127707481384,
+ 0.08744470775127411
+ ],
+ "p_fast": [
+ 0.2956855595111847,
+ 0.2201644331216812,
+ 0.18110233545303345,
+ 0.12301957607269287,
+ 0.09142038226127625,
+ 0.08860767632722855
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9349043965339661,
+ 0.06509559601545334
+ ],
+ "p_stock_bf16": [
+ 0.9339447617530823,
+ 0.06605525314807892
+ ],
+ "p_fast": [
+ 0.9352874159812927,
+ 0.06471257656812668
+ ]
+ },
+ {
+ "state": "router/3",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.978571355342865,
+ 0.021428629755973816
+ ],
+ "p_stock_bf16": [
+ 0.9783018827438354,
+ 0.02169809490442276
+ ],
+ "p_fast": [
+ 0.9786311388015747,
+ 0.02136888913810253
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.3465142548084259,
+ 0.5596334338188171,
+ 0.05834595486521721,
+ 0.03550642356276512
+ ],
+ "p_stock_bf16": [
+ 0.34342753887176514,
+ 0.5640087723731995,
+ 0.05761712044477463,
+ 0.03494655340909958
+ ],
+ "p_fast": [
+ 0.34585127234458923,
+ 0.5591834783554077,
+ 0.05893750116229057,
+ 0.03602777421474457
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.0395582914352417,
+ 0.17612579464912415,
+ 0.14405174553394318,
+ 0.16424626111984253,
+ 0.03720676153898239,
+ 0.4388110935688019
+ ],
+ "p_stock_bf16": [
+ 0.038714081048965454,
+ 0.18183113634586334,
+ 0.1484062522649765,
+ 0.16046540439128876,
+ 0.03694123774766922,
+ 0.43364188075065613
+ ],
+ "p_fast": [
+ 0.0395810641348362,
+ 0.17600956559181213,
+ 0.14365482330322266,
+ 0.161515474319458,
+ 0.037182971835136414,
+ 0.4420561194419861
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8548730611801147,
+ 0.14512692391872406
+ ],
+ "p_stock_bf16": [
+ 0.8590735793113708,
+ 0.14092634618282318
+ ],
+ "p_fast": [
+ 0.8597815036773682,
+ 0.14021848142147064
+ ]
+ },
+ {
+ "state": "router/4",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9914765954017639,
+ 0.008523386903107166
+ ],
+ "p_stock_bf16": [
+ 0.9918753504753113,
+ 0.008124708198010921
+ ],
+ "p_fast": [
+ 0.9910492897033691,
+ 0.008950712159276009
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.18917544186115265,
+ 0.5456979870796204,
+ 0.18691468238830566,
+ 0.07821187376976013
+ ],
+ "p_stock_bf16": [
+ 0.18987496197223663,
+ 0.5446136593818665,
+ 0.18766285479068756,
+ 0.07784846425056458
+ ],
+ "p_fast": [
+ 0.19182297587394714,
+ 0.5453866124153137,
+ 0.18573974072933197,
+ 0.07705070078372955
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.04277270659804344,
+ 0.07839668542146683,
+ 0.0593654103577137,
+ 0.06592525541782379,
+ 0.060230057686567307,
+ 0.6933098435401917
+ ],
+ "p_stock_bf16": [
+ 0.04175952821969986,
+ 0.0756167396903038,
+ 0.05797737091779709,
+ 0.0622006356716156,
+ 0.05889038369059563,
+ 0.7035552859306335
+ ],
+ "p_fast": [
+ 0.04270705208182335,
+ 0.0785502940416336,
+ 0.05929287150502205,
+ 0.0636119619011879,
+ 0.061175037175416946,
+ 0.6946628093719482
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6870533227920532,
+ 0.31294670701026917
+ ],
+ "p_stock_bf16": [
+ 0.6833138465881348,
+ 0.3166861832141876
+ ],
+ "p_fast": [
+ 0.6720089316368103,
+ 0.3279911279678345
+ ]
+ },
+ {
+ "state": "router/5",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.3506668508052826,
+ 0.6493331789970398
+ ],
+ "p_stock_bf16": [
+ 0.3476478159427643,
+ 0.6523521542549133
+ ],
+ "p_fast": [
+ 0.3526475429534912,
+ 0.6473524570465088
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.1377384066581726,
+ 0.4004904329776764,
+ 0.2280898541212082,
+ 0.233681321144104
+ ],
+ "p_stock_bf16": [
+ 0.14049118757247925,
+ 0.4054839015007019,
+ 0.22656910121440887,
+ 0.22745585441589355
+ ],
+ "p_fast": [
+ 0.1420360654592514,
+ 0.40600845217704773,
+ 0.22597773373126984,
+ 0.22597773373126984
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.626227617263794,
+ 0.12996284663677216,
+ 0.11615025252103806,
+ 0.05005913972854614,
+ 0.03461715579032898,
+ 0.0429830364882946
+ ],
+ "p_stock_bf16": [
+ 0.6323071718215942,
+ 0.1274612993001938,
+ 0.11425556242465973,
+ 0.04875829443335533,
+ 0.03351105377078056,
+ 0.04370665177702904
+ ],
+ "p_fast": [
+ 0.6320070624351501,
+ 0.12915457785129547,
+ 0.11287084966897964,
+ 0.048735152930021286,
+ 0.03455839678645134,
+ 0.042673930525779724
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9017393589019775,
+ 0.09826064109802246
+ ],
+ "p_stock_bf16": [
+ 0.9023517370223999,
+ 0.09764823317527771
+ ],
+ "p_fast": [
+ 0.8981428742408752,
+ 0.10185714811086655
+ ]
+ },
+ {
+ "state": "router/6",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9524552822113037,
+ 0.0475446842610836
+ ],
+ "p_stock_bf16": [
+ 0.9531880021095276,
+ 0.0468120314180851
+ ],
+ "p_fast": [
+ 0.9558166265487671,
+ 0.04418332129716873
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0750429704785347,
+ 0.42906996607780457,
+ 0.3325490951538086,
+ 0.16333794593811035
+ ],
+ "p_stock_bf16": [
+ 0.07293495535850525,
+ 0.423830509185791,
+ 0.3379071354866028,
+ 0.16532735526561737
+ ],
+ "p_fast": [
+ 0.07295849919319153,
+ 0.42396730184555054,
+ 0.33801618218421936,
+ 0.16505800187587738
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.06006060540676117,
+ 0.35301893949508667,
+ 0.11654307693243027,
+ 0.2826351225376129,
+ 0.06683231890201569,
+ 0.12090993672609329
+ ],
+ "p_stock_bf16": [
+ 0.06132664158940315,
+ 0.348798930644989,
+ 0.11890487372875214,
+ 0.2835715115070343,
+ 0.06709154695272446,
+ 0.12030648440122604
+ ],
+ "p_fast": [
+ 0.06213318184018135,
+ 0.3506360948085785,
+ 0.11976484209299088,
+ 0.2762945890426636,
+ 0.0690443366765976,
+ 0.1221269890666008
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9648438096046448,
+ 0.03515619784593582
+ ],
+ "p_stock_bf16": [
+ 0.9645892381668091,
+ 0.03541072830557823
+ ],
+ "p_fast": [
+ 0.9653158187866211,
+ 0.03468417003750801
+ ]
+ },
+ {
+ "state": "router/7",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8355194926261902,
+ 0.1644805520772934
+ ],
+ "p_stock_bf16": [
+ 0.8134608268737793,
+ 0.18653921782970428
+ ],
+ "p_fast": [
+ 0.8362873792648315,
+ 0.16371268033981323
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.01463442575186491,
+ 0.15192097425460815,
+ 0.582795262336731,
+ 0.2506493330001831
+ ],
+ "p_stock_bf16": [
+ 0.01527392491698265,
+ 0.15669043362140656,
+ 0.5776450634002686,
+ 0.25039058923721313
+ ],
+ "p_fast": [
+ 0.014540980570018291,
+ 0.15122485160827637,
+ 0.5853930115699768,
+ 0.24884112179279327
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.32057663798332214,
+ 0.10239028930664062,
+ 0.12426351010799408,
+ 0.1029951274394989,
+ 0.04367315396666527,
+ 0.30610138177871704
+ ],
+ "p_stock_bf16": [
+ 0.31915172934532166,
+ 0.10483475774526596,
+ 0.12352553009986877,
+ 0.09925554692745209,
+ 0.04268927872180939,
+ 0.31054314970970154
+ ],
+ "p_fast": [
+ 0.3163244426250458,
+ 0.10309746116399765,
+ 0.12435926496982574,
+ 0.09992548823356628,
+ 0.04365420714020729,
+ 0.31263917684555054
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8969318270683289,
+ 0.10306819528341293
+ ],
+ "p_stock_bf16": [
+ 0.8962059617042542,
+ 0.10379403829574585
+ ],
+ "p_fast": [
+ 0.8993868231773376,
+ 0.10061320662498474
+ ]
+ },
+ {
+ "state": "router/8",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8312171697616577,
+ 0.16878283023834229
+ ],
+ "p_stock_bf16": [
+ 0.8335957527160645,
+ 0.16640429198741913
+ ],
+ "p_fast": [
+ 0.8291464447975159,
+ 0.17085357010364532
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.17582300305366516,
+ 0.5501419305801392,
+ 0.18685142695903778,
+ 0.08718368411064148
+ ],
+ "p_stock_bf16": [
+ 0.17811064422130585,
+ 0.5464804768562317,
+ 0.18885859847068787,
+ 0.0865502581000328
+ ],
+ "p_fast": [
+ 0.17552679777145386,
+ 0.5449010729789734,
+ 0.19053253531455994,
+ 0.08903956413269043
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.17655961215496063,
+ 0.0788390040397644,
+ 0.27396804094314575,
+ 0.13127033412456512,
+ 0.04060972481966019,
+ 0.2987532913684845
+ ],
+ "p_stock_bf16": [
+ 0.17438754439353943,
+ 0.07891026139259338,
+ 0.276231974363327,
+ 0.13163462281227112,
+ 0.04030342027544975,
+ 0.2985321879386902
+ ],
+ "p_fast": [
+ 0.18130624294281006,
+ 0.07951684296131134,
+ 0.2778122127056122,
+ 0.12906856834888458,
+ 0.04029717296361923,
+ 0.2919989228248596
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9413192868232727,
+ 0.05868071690201759
+ ],
+ "p_stock_bf16": [
+ 0.9404625296592712,
+ 0.059537485241889954
+ ],
+ "p_fast": [
+ 0.9430335760116577,
+ 0.056966383010149
+ ]
+ },
+ {
+ "state": "router/9",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9728504419326782,
+ 0.027149531990289688
+ ],
+ "p_stock_bf16": [
+ 0.9725192785263062,
+ 0.02748068980872631
+ ],
+ "p_fast": [
+ 0.9734434485435486,
+ 0.02655657008290291
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.0297623872756958,
+ 0.1067776158452034,
+ 0.4541211724281311,
+ 0.4093388319015503
+ ],
+ "p_stock_bf16": [
+ 0.03031753934919834,
+ 0.10832802206277847,
+ 0.45252877473831177,
+ 0.40882566571235657
+ ],
+ "p_fast": [
+ 0.02990320697426796,
+ 0.10713491588830948,
+ 0.4533732831478119,
+ 0.40958860516548157
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.6785348057746887,
+ 0.12510277330875397,
+ 0.041234564036130905,
+ 0.04308178648352623,
+ 0.07209224253892899,
+ 0.039953723549842834
+ ],
+ "p_stock_bf16": [
+ 0.6748346090316772,
+ 0.12729394435882568,
+ 0.041977085173130035,
+ 0.04330958425998688,
+ 0.07252989709377289,
+ 0.0400548130273819
+ ],
+ "p_fast": [
+ 0.6796230673789978,
+ 0.12473927438259125,
+ 0.04113465175032616,
+ 0.04244040325284004,
+ 0.07219355553388596,
+ 0.039869070053100586
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5614522695541382,
+ 0.4385477602481842
+ ],
+ "p_stock_bf16": [
+ 0.5628975033760071,
+ 0.4371025562286377
+ ],
+ "p_fast": [
+ 0.5589289665222168,
+ 0.4410710036754608
+ ]
+ },
+ {
+ "state": "router/10",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9627551436424255,
+ 0.03724486753344536
+ ],
+ "p_stock_bf16": [
+ 0.9636433720588684,
+ 0.036356665194034576
+ ],
+ "p_fast": [
+ 0.9635062217712402,
+ 0.03649376332759857
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "difficulty",
+ "type": "score",
+ "k": 4,
+ "p_fp32": [
+ 0.08946849405765533,
+ 0.30836138129234314,
+ 0.4054504334926605,
+ 0.1967197209596634
+ ],
+ "p_stock_bf16": [
+ 0.08945710211992264,
+ 0.3067953586578369,
+ 0.40643784403800964,
+ 0.1973097175359726
+ ],
+ "p_fast": [
+ 0.08953835070133209,
+ 0.3088786005973816,
+ 0.40601325035095215,
+ 0.1955697387456894
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "domain",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.14360684156417847,
+ 0.23243853449821472,
+ 0.1422262340784073,
+ 0.07544311136007309,
+ 0.06089730188250542,
+ 0.34538793563842773
+ ],
+ "p_stock_bf16": [
+ 0.14108480513095856,
+ 0.23535139858722687,
+ 0.14330655336380005,
+ 0.07492957264184952,
+ 0.06020760163664818,
+ 0.34512004256248474
+ ],
+ "p_fast": [
+ 0.14650195837020874,
+ 0.23319679498672485,
+ 0.14536188542842865,
+ 0.07541275024414062,
+ 0.0615500882267952,
+ 0.33797651529312134
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "needs_tools",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9559788107872009,
+ 0.04402123764157295
+ ],
+ "p_stock_bf16": [
+ 0.9567959308624268,
+ 0.04320403188467026
+ ],
+ "p_fast": [
+ 0.9566342234611511,
+ 0.043365802615880966
+ ]
+ },
+ {
+ "state": "router/11",
+ "question": "is_sensitive",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9810004234313965,
+ 0.018999604508280754
+ ],
+ "p_stock_bf16": [
+ 0.9810219407081604,
+ 0.01897803694009781
+ ],
+ "p_fast": [
+ 0.9814534187316895,
+ 0.018546564504504204
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.8675079345703125,
+ 0.01611124537885189,
+ 0.023573242127895355,
+ 0.039238445460796356,
+ 0.017879746854305267,
+ 0.03568948805332184
+ ],
+ "p_stock_bf16": [
+ 0.8677341341972351,
+ 0.01601775549352169,
+ 0.02367270737886429,
+ 0.039029695093631744,
+ 0.017730040475726128,
+ 0.03581566736102104
+ ],
+ "p_fast": [
+ 0.8689409494400024,
+ 0.01616583578288555,
+ 0.02315649203956127,
+ 0.03908397629857063,
+ 0.01734340935945511,
+ 0.035309433937072754
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7418201565742493,
+ 0.2581798732280731
+ ],
+ "p_stock_bf16": [
+ 0.7485361695289612,
+ 0.2514638304710388
+ ],
+ "p_fast": [
+ 0.7455373406410217,
+ 0.2544626295566559
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.707365870475769,
+ 0.29263415932655334
+ ],
+ "p_stock_bf16": [
+ 0.7023259401321411,
+ 0.2976740896701813
+ ],
+ "p_fast": [
+ 0.7075057625770569,
+ 0.2924942374229431
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.006834388244897127,
+ 0.3354489803314209,
+ 0.657716691493988
+ ],
+ "p_stock_bf16": [
+ 0.006754753179848194,
+ 0.33579349517822266,
+ 0.6574517488479614
+ ],
+ "p_fast": [
+ 0.006772450637072325,
+ 0.33405327796936035,
+ 0.6591742634773254
+ ]
+ },
+ {
+ "state": "email/0",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8669513463973999,
+ 0.1330486387014389
+ ],
+ "p_stock_bf16": [
+ 0.8696039319038391,
+ 0.13039611279964447
+ ],
+ "p_fast": [
+ 0.8678218126296997,
+ 0.1321781426668167
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.006087132263928652,
+ 0.7426608204841614,
+ 0.007194834761321545,
+ 0.19574464857578278,
+ 0.008362685330212116,
+ 0.03994989022612572
+ ],
+ "p_stock_bf16": [
+ 0.006005673203617334,
+ 0.7389310002326965,
+ 0.007131913676857948,
+ 0.19965898990631104,
+ 0.008338051848113537,
+ 0.03993431106209755
+ ],
+ "p_fast": [
+ 0.006093958858400583,
+ 0.7381690144538879,
+ 0.00712455902248621,
+ 0.20023371279239655,
+ 0.008329452946782112,
+ 0.04004926607012749
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6881608366966248,
+ 0.31183910369873047
+ ],
+ "p_stock_bf16": [
+ 0.681090772151947,
+ 0.3189091980457306
+ ],
+ "p_fast": [
+ 0.6893056035041809,
+ 0.3106943964958191
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7670397758483887,
+ 0.23296020925045013
+ ],
+ "p_stock_bf16": [
+ 0.7652426362037659,
+ 0.23475733399391174
+ ],
+ "p_fast": [
+ 0.7543725371360779,
+ 0.24562746286392212
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.0042326608672738075,
+ 0.1721258908510208,
+ 0.8236414790153503
+ ],
+ "p_stock_bf16": [
+ 0.004192738328129053,
+ 0.17144997417926788,
+ 0.8243573307991028
+ ],
+ "p_fast": [
+ 0.004244131967425346,
+ 0.17423084378242493,
+ 0.821524977684021
+ ]
+ },
+ {
+ "state": "email/1",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6243698596954346,
+ 0.3756301701068878
+ ],
+ "p_stock_bf16": [
+ 0.6332729458808899,
+ 0.3667270243167877
+ ],
+ "p_fast": [
+ 0.6210095286369324,
+ 0.3789904713630676
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.11010496318340302,
+ 0.0191094558686018,
+ 0.48997482657432556,
+ 0.021628309041261673,
+ 0.034384146332740784,
+ 0.3247982859611511
+ ],
+ "p_stock_bf16": [
+ 0.10902542620897293,
+ 0.018982818350195885,
+ 0.4857634902000427,
+ 0.021342957392334938,
+ 0.03491474688053131,
+ 0.32997044920921326
+ ],
+ "p_fast": [
+ 0.10900764912366867,
+ 0.019354064017534256,
+ 0.4875851571559906,
+ 0.021931026130914688,
+ 0.034772954881191254,
+ 0.3273491859436035
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9472465515136719,
+ 0.052753400057554245
+ ],
+ "p_stock_bf16": [
+ 0.9483462572097778,
+ 0.05165378004312515
+ ],
+ "p_fast": [
+ 0.9469902515411377,
+ 0.0530097670853138
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9539501070976257,
+ 0.046049848198890686
+ ],
+ "p_stock_bf16": [
+ 0.9553191661834717,
+ 0.044680871069431305
+ ],
+ "p_fast": [
+ 0.9548162817955017,
+ 0.045183759182691574
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.5677735209465027,
+ 0.3914264142513275,
+ 0.0408000610768795
+ ],
+ "p_stock_bf16": [
+ 0.566714882850647,
+ 0.3925519287586212,
+ 0.040733207017183304
+ ],
+ "p_fast": [
+ 0.5665338635444641,
+ 0.3924265503883362,
+ 0.04103957861661911
+ ]
+ },
+ {
+ "state": "email/2",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5804012417793274,
+ 0.4195987582206726
+ ],
+ "p_stock_bf16": [
+ 0.5780909061431885,
+ 0.42190903425216675
+ ],
+ "p_fast": [
+ 0.5812140107154846,
+ 0.41878604888916016
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.02506846934556961,
+ 0.08374807238578796,
+ 0.03477940708398819,
+ 0.13404075801372528,
+ 0.06331031024456024,
+ 0.6590529680252075
+ ],
+ "p_stock_bf16": [
+ 0.025366846472024918,
+ 0.08285042643547058,
+ 0.03521838039159775,
+ 0.1341843456029892,
+ 0.06302927434444427,
+ 0.6593506932258606
+ ],
+ "p_fast": [
+ 0.025099223479628563,
+ 0.08326728641986847,
+ 0.03457564115524292,
+ 0.13545344769954681,
+ 0.06409311294555664,
+ 0.657511293888092
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5329158306121826,
+ 0.46708419919013977
+ ],
+ "p_stock_bf16": [
+ 0.5299932360649109,
+ 0.4700067639350891
+ ],
+ "p_fast": [
+ 0.5324251055717468,
+ 0.4675748944282532
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6222484707832336,
+ 0.37775152921676636
+ ],
+ "p_stock_bf16": [
+ 0.6238353848457336,
+ 0.37616467475891113
+ ],
+ "p_fast": [
+ 0.6252092719078064,
+ 0.3747907280921936
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.2441190630197525,
+ 0.41501176357269287,
+ 0.3408691883087158
+ ],
+ "p_stock_bf16": [
+ 0.24969273805618286,
+ 0.41167372465133667,
+ 0.33863356709480286
+ ],
+ "p_fast": [
+ 0.2520322799682617,
+ 0.4074938893318176,
+ 0.34047389030456543
+ ]
+ },
+ {
+ "state": "email/3",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8240371942520142,
+ 0.17596276104450226
+ ],
+ "p_stock_bf16": [
+ 0.8302159309387207,
+ 0.1697840839624405
+ ],
+ "p_fast": [
+ 0.8245325684547424,
+ 0.17546740174293518
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.02197432890534401,
+ 0.03848302364349365,
+ 0.018150469288229942,
+ 0.042898982763290405,
+ 0.05884347856044769,
+ 0.8196496963500977
+ ],
+ "p_stock_bf16": [
+ 0.021559210494160652,
+ 0.03813435509800911,
+ 0.017873212695121765,
+ 0.04254194721579552,
+ 0.05814794823527336,
+ 0.8217433094978333
+ ],
+ "p_fast": [
+ 0.021548829972743988,
+ 0.038716234266757965,
+ 0.017864609137177467,
+ 0.04285496473312378,
+ 0.05766766145825386,
+ 0.821347713470459
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6074861884117126,
+ 0.39251381158828735
+ ],
+ "p_stock_bf16": [
+ 0.6130049228668213,
+ 0.3869951069355011
+ ],
+ "p_fast": [
+ 0.6085940003395081,
+ 0.39140594005584717
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8024953007698059,
+ 0.1975046694278717
+ ],
+ "p_stock_bf16": [
+ 0.8019358515739441,
+ 0.1980641782283783
+ ],
+ "p_fast": [
+ 0.8056321740150452,
+ 0.19436781108379364
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.07369204610586166,
+ 0.6068594455718994,
+ 0.3194485306739807
+ ],
+ "p_stock_bf16": [
+ 0.07142382115125656,
+ 0.6122061014175415,
+ 0.31637006998062134
+ ],
+ "p_fast": [
+ 0.07342600077390671,
+ 0.6076259613037109,
+ 0.31894806027412415
+ ]
+ },
+ {
+ "state": "email/4",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.42933106422424316,
+ 0.5706689953804016
+ ],
+ "p_stock_bf16": [
+ 0.42942938208580017,
+ 0.5705706477165222
+ ],
+ "p_fast": [
+ 0.4234580099582672,
+ 0.5765419602394104
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.03612571209669113,
+ 0.04793607071042061,
+ 0.1972629874944687,
+ 0.59276282787323,
+ 0.017160922288894653,
+ 0.10875151306390762
+ ],
+ "p_stock_bf16": [
+ 0.03650587424635887,
+ 0.048551712185144424,
+ 0.19889670610427856,
+ 0.5891749262809753,
+ 0.017109958454966545,
+ 0.10976076871156693
+ ],
+ "p_fast": [
+ 0.03623977676033974,
+ 0.048386454582214355,
+ 0.19821973145008087,
+ 0.5894676446914673,
+ 0.017252719029784203,
+ 0.1104336753487587
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.13375742733478546,
+ 0.866242527961731
+ ],
+ "p_stock_bf16": [
+ 0.13330236077308655,
+ 0.8666976690292358
+ ],
+ "p_fast": [
+ 0.13511791825294495,
+ 0.8648820519447327
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.18096666038036346,
+ 0.8190333247184753
+ ],
+ "p_stock_bf16": [
+ 0.18097351491451263,
+ 0.8190264105796814
+ ],
+ "p_fast": [
+ 0.1792430281639099,
+ 0.8207569718360901
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.06192639842629433,
+ 0.5973776578903198,
+ 0.34069591760635376
+ ],
+ "p_stock_bf16": [
+ 0.06067280471324921,
+ 0.6009215116500854,
+ 0.3384057283401489
+ ],
+ "p_fast": [
+ 0.06268815696239471,
+ 0.5970965027809143,
+ 0.3402153551578522
+ ]
+ },
+ {
+ "state": "email/5",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6549398899078369,
+ 0.3450601100921631
+ ],
+ "p_stock_bf16": [
+ 0.6570984721183777,
+ 0.3429014980792999
+ ],
+ "p_fast": [
+ 0.646683394908905,
+ 0.35331663489341736
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.025907106697559357,
+ 0.2321919947862625,
+ 0.018995536491274834,
+ 0.30921074748039246,
+ 0.10555917769670486,
+ 0.308135449886322
+ ],
+ "p_stock_bf16": [
+ 0.026571763679385185,
+ 0.22620701789855957,
+ 0.018989982083439827,
+ 0.30648279190063477,
+ 0.11165300011634827,
+ 0.31009548902511597
+ ],
+ "p_fast": [
+ 0.0263910423964262,
+ 0.22466853260993958,
+ 0.0188608281314373,
+ 0.324030339717865,
+ 0.1051972359418869,
+ 0.3008520007133484
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.48461663722991943,
+ 0.5153834223747253
+ ],
+ "p_stock_bf16": [
+ 0.46429428458213806,
+ 0.5357056856155396
+ ],
+ "p_fast": [
+ 0.4675748944282532,
+ 0.5324251055717468
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.5290076732635498,
+ 0.4709923565387726
+ ],
+ "p_stock_bf16": [
+ 0.5180585384368896,
+ 0.48194143176078796
+ ],
+ "p_fast": [
+ 0.5319388508796692,
+ 0.4680611491203308
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.008156470023095608,
+ 0.3449278473854065,
+ 0.6469156742095947
+ ],
+ "p_stock_bf16": [
+ 0.008759923279285431,
+ 0.35265836119651794,
+ 0.6385817527770996
+ ],
+ "p_fast": [
+ 0.008696337230503559,
+ 0.34737399220466614,
+ 0.643929660320282
+ ]
+ },
+ {
+ "state": "email/6",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.0917402058839798,
+ 0.9082598686218262
+ ],
+ "p_stock_bf16": [
+ 0.08756384253501892,
+ 0.9124361872673035
+ ],
+ "p_fast": [
+ 0.08977328985929489,
+ 0.9102267622947693
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.5252329111099243,
+ 0.022251885384321213,
+ 0.12854264676570892,
+ 0.017590103670954704,
+ 0.03141535073518753,
+ 0.27496716380119324
+ ],
+ "p_stock_bf16": [
+ 0.5248392224311829,
+ 0.022003840655088425,
+ 0.1260065734386444,
+ 0.017136607319116592,
+ 0.03127375990152359,
+ 0.27873995900154114
+ ],
+ "p_fast": [
+ 0.544701874256134,
+ 0.022133972495794296,
+ 0.12357400357723236,
+ 0.01750941388309002,
+ 0.030729975551366806,
+ 0.2613508105278015
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9589805006980896,
+ 0.041019506752491
+ ],
+ "p_stock_bf16": [
+ 0.960212230682373,
+ 0.039787814021110535
+ ],
+ "p_fast": [
+ 0.960212230682373,
+ 0.039787814021110535
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9364396929740906,
+ 0.06356034427881241
+ ],
+ "p_stock_bf16": [
+ 0.9343951344490051,
+ 0.06560483574867249
+ ],
+ "p_fast": [
+ 0.9317114353179932,
+ 0.06828853487968445
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.24989286065101624,
+ 0.572486162185669,
+ 0.1776210069656372
+ ],
+ "p_stock_bf16": [
+ 0.2501170039176941,
+ 0.5725234746932983,
+ 0.17735949158668518
+ ],
+ "p_fast": [
+ 0.25235694646835327,
+ 0.5686951875686646,
+ 0.17894788086414337
+ ]
+ },
+ {
+ "state": "email/7",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.23296482861042023,
+ 0.767035186290741
+ ],
+ "p_stock_bf16": [
+ 0.2359876036643982,
+ 0.7640123963356018
+ ],
+ "p_fast": [
+ 0.22627031803131104,
+ 0.773729681968689
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.12097876518964767,
+ 0.22702394425868988,
+ 0.050346773117780685,
+ 0.07463028281927109,
+ 0.23697327077388763,
+ 0.29004696011543274
+ ],
+ "p_stock_bf16": [
+ 0.11722850799560547,
+ 0.24006791412830353,
+ 0.051013678312301636,
+ 0.07480662316083908,
+ 0.22274592518806458,
+ 0.2941373884677887
+ ],
+ "p_fast": [
+ 0.11071379482746124,
+ 0.2267267256975174,
+ 0.049128953367471695,
+ 0.06820879876613617,
+ 0.25466689467430115,
+ 0.29055482149124146
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.8749845623970032,
+ 0.1250154972076416
+ ],
+ "p_stock_bf16": [
+ 0.8763369917869568,
+ 0.12366295605897903
+ ],
+ "p_fast": [
+ 0.8828703761100769,
+ 0.1171296015381813
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.6822959780693054,
+ 0.3177039921283722
+ ],
+ "p_stock_bf16": [
+ 0.6794977784156799,
+ 0.3205021917819977
+ ],
+ "p_fast": [
+ 0.6828910112380981,
+ 0.31710895895957947
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.004992176312953234,
+ 0.4028315842151642,
+ 0.5921761989593506
+ ],
+ "p_stock_bf16": [
+ 0.0049452343955636024,
+ 0.4053190350532532,
+ 0.5897356867790222
+ ],
+ "p_fast": [
+ 0.004990866873413324,
+ 0.409059077501297,
+ 0.5859500765800476
+ ]
+ },
+ {
+ "state": "email/8",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7262336015701294,
+ 0.2737663686275482
+ ],
+ "p_stock_bf16": [
+ 0.7171109914779663,
+ 0.28288906812667847
+ ],
+ "p_fast": [
+ 0.7337858080863953,
+ 0.26621416211128235
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.073619544506073,
+ 0.07772894203662872,
+ 0.05636807158589363,
+ 0.11553928256034851,
+ 0.042433951050043106,
+ 0.634310245513916
+ ],
+ "p_stock_bf16": [
+ 0.07379376143217087,
+ 0.07794173806905746,
+ 0.05570245161652565,
+ 0.11519041657447815,
+ 0.042376190423965454,
+ 0.6349955201148987
+ ],
+ "p_fast": [
+ 0.07130584865808487,
+ 0.07590467482805252,
+ 0.05467208847403526,
+ 0.10746169835329056,
+ 0.042578667402267456,
+ 0.6480770111083984
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.39480140805244446,
+ 0.6051985621452332
+ ],
+ "p_stock_bf16": [
+ 0.39361807703971863,
+ 0.606381893157959
+ ],
+ "p_fast": [
+ 0.4050956666469574,
+ 0.594904363155365
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7475924491882324,
+ 0.2524075210094452
+ ],
+ "p_stock_bf16": [
+ 0.7514657974243164,
+ 0.2485341727733612
+ ],
+ "p_fast": [
+ 0.7514657974243164,
+ 0.2485341727733612
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.13149239122867584,
+ 0.46978768706321716,
+ 0.39871999621391296
+ ],
+ "p_stock_bf16": [
+ 0.13033418357372284,
+ 0.4721096158027649,
+ 0.3975561857223511
+ ],
+ "p_fast": [
+ 0.1293201893568039,
+ 0.468436598777771,
+ 0.4022432267665863
+ ]
+ },
+ {
+ "state": "email/9",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7810052633285522,
+ 0.21899472177028656
+ ],
+ "p_stock_bf16": [
+ 0.780369758605957,
+ 0.21963027119636536
+ ],
+ "p_fast": [
+ 0.7868686318397522,
+ 0.21313141286373138
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.006574548780918121,
+ 0.875615656375885,
+ 0.007916916161775589,
+ 0.028746768832206726,
+ 0.014641680754721165,
+ 0.06650436669588089
+ ],
+ "p_stock_bf16": [
+ 0.006495168432593346,
+ 0.8777033090591431,
+ 0.007834669202566147,
+ 0.028213728219270706,
+ 0.014410161413252354,
+ 0.06534313410520554
+ ],
+ "p_fast": [
+ 0.006493290886282921,
+ 0.877449631690979,
+ 0.007832405157387257,
+ 0.028205573558807373,
+ 0.014182652346789837,
+ 0.0658365935087204
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9809960126876831,
+ 0.01900395005941391
+ ],
+ "p_stock_bf16": [
+ 0.9809491038322449,
+ 0.019050899893045425
+ ],
+ "p_fast": [
+ 0.9813107252120972,
+ 0.018689310178160667
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9931678175926208,
+ 0.006832200102508068
+ ],
+ "p_stock_bf16": [
+ 0.9930961728096008,
+ 0.006903838366270065
+ ],
+ "p_fast": [
+ 0.9932025074958801,
+ 0.00679753115400672
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.01595943048596382,
+ 0.36753395199775696,
+ 0.6165065765380859
+ ],
+ "p_stock_bf16": [
+ 0.016161473467946053,
+ 0.3678334057331085,
+ 0.6160050630569458
+ ],
+ "p_fast": [
+ 0.016037719324231148,
+ 0.3678796887397766,
+ 0.6160825490951538
+ ]
+ },
+ {
+ "state": "email/10",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.44522884488105774,
+ 0.5547711253166199
+ ],
+ "p_stock_bf16": [
+ 0.4468567669391632,
+ 0.5531432628631592
+ ],
+ "p_fast": [
+ 0.45919740200042725,
+ 0.5408025979995728
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "category",
+ "type": "choice",
+ "k": 6,
+ "p_fp32": [
+ 0.9162148833274841,
+ 0.013147835619747639,
+ 0.015526391565799713,
+ 0.0067847673781216145,
+ 0.008260440081357956,
+ 0.04006573557853699
+ ],
+ "p_stock_bf16": [
+ 0.9171636700630188,
+ 0.01287981029599905,
+ 0.015415113419294357,
+ 0.00668195728212595,
+ 0.008186905644834042,
+ 0.0396726056933403
+ ],
+ "p_fast": [
+ 0.9164187908172607,
+ 0.013072011061012745,
+ 0.015282729640603065,
+ 0.006781670264899731,
+ 0.008180256932973862,
+ 0.04026462882757187
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "is_spam",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9984169006347656,
+ 0.0015830977354198694
+ ],
+ "p_stock_bf16": [
+ 0.9984330534934998,
+ 0.0015669887652620673
+ ],
+ "p_fast": [
+ 0.9983707070350647,
+ 0.0016293087974190712
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "is_phishing",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.9993016719818115,
+ 0.0006983057828620076
+ ],
+ "p_stock_bf16": [
+ 0.9992792010307312,
+ 0.0007208379684016109
+ ],
+ "p_fast": [
+ 0.9992848038673401,
+ 0.0007152324542403221
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "urgency",
+ "type": "score",
+ "k": 3,
+ "p_fp32": [
+ 0.29234760999679565,
+ 0.47026416659355164,
+ 0.23738817870616913
+ ],
+ "p_stock_bf16": [
+ 0.291825532913208,
+ 0.4699934124946594,
+ 0.23818109929561615
+ ],
+ "p_fast": [
+ 0.29236745834350586,
+ 0.4708661735057831,
+ 0.23676638305187225
+ ]
+ },
+ {
+ "state": "email/11",
+ "question": "needs_reply",
+ "type": "noul",
+ "k": 2,
+ "p_fp32": [
+ 0.7040170431137085,
+ 0.2959829270839691
+ ],
+ "p_stock_bf16": [
+ 0.7077330946922302,
+ 0.29226693511009216
+ ],
+ "p_fast": [
+ 0.7023770213127136,
+ 0.29762303829193115
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/laya/agent.py b/laya/agent.py
index a76041f7..f55721ea 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -440,9 +440,11 @@ class Agent(HookRegistry):
def accelerate(self, use_graphs: bool = True, strict: bool = False):
"""Replace the model forward with the TileLang fast path (fused GEMM/GEGLU/LayerNorm/RoPE kernels,
- sliding-window flash attention, bf16 resident weights, CUDA graphs per shape bucket).
+ sliding-window flash attention, 16-bit resident weights, CUDA graphs per shape bucket).
- Same numerics as the stock bf16 autocast path (see benchmarks/bench_fast.py). Returns True if
+ The fast path runs in the agent's autocast dtype at the time of the call (bf16 or fp16), so it
+ matches the stock forward it replaces within rounding (see benchmarks/parity_fast.py). After
+ changing `agent.dtype`, call `deaccelerate()` then `accelerate()` to rebuild it. Returns True if
enabled. With `strict=False` any failure (no CUDA, tilelang missing) leaves the stock path in place.
"""
if self._fast is not None:
@@ -455,7 +457,9 @@ class Agent(HookRegistry):
for _attempt in range(2): # tilelang's JIT cache has been seen to fail once, then succeed
try:
from .fast import FastLaya
- self._fast = FastLaya(self.model, max_len=self.cfg.get("max_len", 512), use_graphs=use_graphs)
+ fast_dtype = self.dtype if self.dtype in (torch.bfloat16, torch.float16) else torch.bfloat16
+ self._fast = FastLaya(self.model, max_len=self.cfg.get("max_len", 512), use_graphs=use_graphs,
+ dtype=fast_dtype)
break
except Exception as e: # tilelang missing / unsupported arch
last = e
diff --git a/laya/fast.py b/laya/fast.py
index c960134c..7333ee57 100644
--- a/laya/fast.py
+++ b/laya/fast.py
@@ -1,4 +1,4 @@
-"""GPU fast path for laya: TileLang fused kernels + bf16 resident weights + CUDA graphs.
+"""GPU fast path for laya: TileLang fused kernels + 16-bit resident weights + CUDA graphs.
agent = laya.load("convaiinnovations/laya", fast=True) # or agent.accelerate()
@@ -9,7 +9,7 @@ import threading
import torch
from . import tl_kernels as K
-BF = torch.bfloat16
+_KERNEL_DTYPES = {torch.bfloat16: "bfloat16", torch.float16: "float16"}
def _top_two(probs):
@@ -25,7 +25,14 @@ def _bucket_n(n):
class FastLaya:
- def __init__(self, model, max_len=1024, use_graphs=True, verbose=False):
+ def __init__(self, model, max_len=1024, use_graphs=True, verbose=False, dtype=torch.bfloat16):
+ """`dtype` is the 16-bit type of the weights and the activations between kernels (the residual stream
+ and every accumulation stay fp32). Pass the agent's autocast dtype so the fast path runs in the same
+ precision as the stock forward it replaces: bf16 (the shipped checkpoints' default) or fp16."""
+ if dtype not in _KERNEL_DTYPES:
+ raise ValueError("laya fast path supports torch.bfloat16 and torch.float16, got %s" % (dtype,))
+ self.dtype = dt = dtype
+ kd = _KERNEL_DTYPES[dtype]
self.m = model
enc = model.encoder
cfg = enc.config
@@ -42,7 +49,7 @@ class FastLaya:
self.eps = cfg.norm_eps
self.max_len = max_len
f32 = lambda t: t.detach().float().contiguous()
- b16 = lambda t: t.detach().to(BF).contiguous()
+ b16 = lambda t: t.detach().to(dt).contiguous()
zeros = torch.zeros(self.D, device=dev)
self.zeros = {self.D: zeros, 3 * self.D: torch.zeros(3 * self.D, device=dev), 4 * self.D: torch.zeros(4 * self.D, device=dev),
2 * self.F: torch.zeros(2 * self.F, device=dev)}
@@ -61,7 +68,7 @@ class FastLaya:
mlp_ln=f32(lyr.mlp_norm.weight), wi=b16(lyr.mlp.Wi.weight), wo2=b16(lyr.mlp.Wo.weight),
window=(win if lyr.attention_type == "sliding_attention" else 0), ltype=lyr.attention_type))
self.final_ln = f32(enc.final_norm.weight)
- # --- rotary tables (rounded through bf16 exactly like HF does before applying)
+ # --- rotary tables (rounded through the 16-bit dtype exactly like HF does before applying)
rot = enc.rotary_emb
pos = torch.arange(max_len, device=dev).float()
self.rope = {}
@@ -69,7 +76,7 @@ class FastLaya:
inv = getattr(rot, f"{lt}_inv_freq").float()
scl = getattr(rot, f"{lt}_attention_scaling")
fr = torch.outer(pos, inv)
- self.rope[lt] = ((fr.cos() * scl).to(BF).float().contiguous(), (fr.sin() * scl).to(BF).float().contiguous())
+ self.rope[lt] = ((fr.cos() * scl).to(dt).float().contiguous(), (fr.sin() * scl).to(dt).float().contiguous())
# --- decision head (nn.TransformerEncoderLayer, norm_first, relu)
self.type_emb = b16(model.type_emb.weight)
self.head = []
@@ -81,17 +88,18 @@ class FastLaya:
l1w=b16(lyr.linear1.weight), l1b=f32(lyr.linear1.bias), l2w=b16(lyr.linear2.weight), l2b=f32(lyr.linear2.bias)))
# --- kernels (M is dynamic, so these compile once)
D, F = self.D, self.F
- self.k_qkv = K.gemm_kernel(3 * D, D)
- self.k_o = K.gemm_kernel(D, D)
- self.k_geglu = K.gemm_geglu_kernel(F, D)
- self.k_o2 = K.gemm_kernel(D, F)
- self.k_addln = K.add_ln_kernel(D, residual=True, bias=False, eps=self.eps)
- self.k_addln_b = K.add_ln_kernel(D, residual=True, bias=True, eps=1e-5)
- self.k_ln_b = K.add_ln_kernel(D, residual=False, bias=True, eps=1e-5)
- self.k_in = K.gemm_kernel(3 * D, D, bias=True)
- self.k_out = K.gemm_kernel(D, D, bias=True)
- self.k_ffn1 = K.gemm_kernel(4 * D, D, bias=True, act="relu")
- self.k_ffn2 = K.gemm_kernel(D, 4 * D, bias=True)
+ self.kdtype = kd
+ self.k_qkv = K.gemm_kernel(3 * D, D, dtype=kd)
+ self.k_o = K.gemm_kernel(D, D, dtype=kd)
+ self.k_geglu = K.gemm_geglu_kernel(F, D, dtype=kd)
+ self.k_o2 = K.gemm_kernel(D, F, dtype=kd)
+ self.k_addln = K.add_ln_kernel(D, residual=True, bias=False, eps=self.eps, dtype=kd)
+ self.k_addln_b = K.add_ln_kernel(D, residual=True, bias=True, eps=1e-5, dtype=kd)
+ self.k_ln_b = K.add_ln_kernel(D, residual=False, bias=True, eps=1e-5, dtype=kd)
+ self.k_in = K.gemm_kernel(3 * D, D, bias=True, dtype=kd)
+ self.k_out = K.gemm_kernel(D, D, bias=True, dtype=kd)
+ self.k_ffn1 = K.gemm_kernel(4 * D, D, bias=True, act="relu", dtype=kd)
+ self.k_ffn2 = K.gemm_kernel(D, 4 * D, bias=True, dtype=kd)
self._rope_k, self._rope_tab, self._attn_k = None, {}, {}
self.graphs = {}
@@ -101,7 +109,7 @@ class FastLaya:
def rope_k(self):
if self._rope_k is None:
- self._rope_k = K.rope_kernel(self.H, self.Dh)
+ self._rope_k = K.rope_kernel(self.H, self.Dh, dtype=self.kdtype)
return self._rope_k
def rope_tab(self, ltype, L):
@@ -114,22 +122,22 @@ class FastLaya:
def attn_k(self, B, L, window):
key = (None, None, window) if L <= self.DYNAMIC_MAX_L else (B, L, window)
if key not in self._attn_k:
- self._attn_k[key] = K.attn_kernel(key[0], key[1], self.H, self.Dh, window=window)
+ self._attn_k[key] = K.attn_kernel(key[0], key[1], self.H, self.Dh, window=window, dtype=self.kdtype)
return self._attn_k[key]
# ------------------------------------------------------------------ encoder + head on padded [B, L]
def _encode(self, ids, lens, qtype):
- """ids [B,L] long (padded), lens [B] int32, qtype [B] long -> hidden [B, L, D] bf16"""
+ """ids [B,L] long (padded), lens [B] int32, qtype [B] long -> hidden [B, L, D] fp32"""
B, L = ids.shape
M, D = B * L, self.D
dev = self.dev
emb = torch.nn.functional.embedding(ids, self.emb_w).view(M, D).float()
# residual stream = embeddings.norm(emb), kept in fp32 exactly like the stock autocast path
X = torch.nn.functional.layer_norm(emb, (D,), self.emb_ln, None, self.eps)
- Y = X.to(BF) # layer 0 attends to it directly (attn_norm = Identity)
- qkv = torch.empty(M, 3 * D, device=dev, dtype=BF)
- O = torch.empty(M, D, device=dev, dtype=BF)
- G = torch.empty(M, self.F, device=dev, dtype=BF)
+ Y = X.to(self.dtype) # layer 0 attends to it directly (attn_norm = Identity)
+ qkv = torch.empty(M, 3 * D, device=dev, dtype=self.dtype)
+ O = torch.empty(M, D, device=dev, dtype=self.dtype)
+ G = torch.empty(M, self.F, device=dev, dtype=self.dtype)
z = self.zeros
nl = len(self.layers)
for i, ly in enumerate(self.layers):
@@ -145,7 +153,7 @@ class FastLaya:
self.k_addln(X, Y, nxt, z[D], Y) # X += Y ; Y = next norm(X)
# decision head: h = final_norm(x) + type_emb ; 2 x pre-norm transformer layers (relu ffn)
X = (Y.view(B, L, D).float() + self.type_emb[qtype].float()[:, None, :]).view(M, D).contiguous() # fp32 stream for the head
- F1 = torch.empty(M, 4 * D, device=dev, dtype=BF)
+ F1 = torch.empty(M, 4 * D, device=dev, dtype=self.dtype)
for j, h in enumerate(self.head):
self.k_ln_b(X, Y, h["n1w"], h["n1b"], Y)
self.k_in(Y, h["in_w"], h["in_b"], qkv)
diff --git a/laya/tl_kernels.py b/laya/tl_kernels.py
index d268a909..b5c9ee61 100644
--- a/laya/tl_kernels.py
+++ b/laya/tl_kernels.py
@@ -1,16 +1,23 @@
"""TileLang kernels for the Laya (ModernBERT + decision head) encoder.
-All kernels take bf16 activations, accumulate in fp32. Row count M is a runtime
+All kernels take 16-bit activations (bf16 by default, fp16 with dtype="float16"), accumulate in fp32. Row count M is a runtime
symbol so one compiled kernel serves every batch/sequence bucket; M must be a
multiple of 16 (the caller pads); out-of-bounds rows are predicated by TileLang.
"""
import tilelang
import tilelang.language as T
-DT, ACC = "bfloat16", "float"
+ACC = "float"
+DTYPES = ("bfloat16", "float16")
FAST = {tilelang.PassConfigKey.TL_ENABLE_FAST_MATH: True}
+def _dt(dtype):
+ if dtype not in DTYPES:
+ raise ValueError("tl_kernels dtype must be one of %s, got %r" % (DTYPES, dtype))
+ return dtype
+
+
def _act(x, kind):
if kind == "gelu": # exact erf-GELU, what HF "gelu" means
return 0.5 * x * (1.0 + T.erf(x * 0.7071067811865476))
@@ -21,8 +28,9 @@ def _act(x, kind):
# ----------------------------------------------------------------------------- GEMM
@tilelang.jit(pass_configs=FAST)
-def gemm_kernel(N, K, bias=False, act="none", bm=64, bn=128, bk=64, stages=3, threads=128):
+def gemm_kernel(N, K, bias=False, act="none", bm=64, bn=128, bk=64, stages=3, threads=128, dtype="bfloat16"):
"""C[M,N] = act(A[M,K] @ W[N,K]^T + b)."""
+ DT = _dt(dtype)
M = T.dynamic("M")
@T.prim_func
@@ -36,18 +44,21 @@ def gemm_kernel(N, K, bias=False, act="none", bm=64, bn=128, bk=64, stages=3, th
T.copy(A[by * bm, k * bk], A_s)
T.copy(W[bx * bn, k * bk], W_s)
T.gemm(A_s, W_s, C_l, transpose_B=True)
- for i, j in T.Parallel(bm, bn):
- v = C_l[i, j]
- if bias:
- v = v + Bv[bx * bn + j]
- C_l[i, j] = _act(v, act)
+ # `bias` is a Python-level flag, so each branch is one expression (no re-bound TileLang value)
+ if bias:
+ for i, j in T.Parallel(bm, bn):
+ C_l[i, j] = _act(C_l[i, j] + Bv[bx * bn + j], act)
+ else:
+ for i, j in T.Parallel(bm, bn):
+ C_l[i, j] = _act(C_l[i, j], act)
T.copy(C_l, C[by * bm, bx * bn])
return main
@tilelang.jit(pass_configs=FAST)
-def gemm_geglu_kernel(F, K, bm=64, bn=64, bk=64, stages=3, threads=128):
+def gemm_geglu_kernel(F, K, bm=64, bn=64, bk=64, stages=3, threads=128, dtype="bfloat16"):
"""ModernBERT GLU MLP up-projection, fused: C[M,F] = gelu(A @ Wi[:F]^T) * (A @ Wi[F:]^T)."""
+ DT = _dt(dtype)
M = T.dynamic("M")
@T.prim_func
@@ -73,11 +84,12 @@ def gemm_geglu_kernel(F, K, bm=64, bn=64, bk=64, stages=3, threads=128):
# ----------------------------------------------------------------------------- LayerNorm (+residual)
@tilelang.jit(pass_configs=FAST)
-def add_ln_kernel(D, residual=True, bias=False, eps=1e-5, bm=4, threads=32):
- """X (fp32 residual stream) += R (bf16 branch output, if residual); Y (bf16) = LN(X) * w (+ b).
+def add_ln_kernel(D, residual=True, bias=False, eps=1e-5, bm=4, threads=32, dtype="bfloat16"):
+ """X (fp32 residual stream) += R (16-bit branch output, if residual); Y (16-bit) = LN(X) * w (+ b).
The residual stream stays in fp32 exactly like the stock autocast path: ModernBERT-large's residual
activations reach ~3e4, where bf16's 8-bit mantissa would lose ~100 units per add and drift layer by layer."""
+ DT = _dt(dtype)
M = T.dynamic("M")
@T.prim_func
@@ -108,11 +120,12 @@ def add_ln_kernel(D, residual=True, bias=False, eps=1e-5, bm=4, threads=32):
T.reduce_sum(xs, var, dim=1)
for i in T.Parallel(bm):
var[i] = T.rsqrt(var[i] / D + eps)
- for i, j in T.Parallel(bm, D):
- v = (x[i, j] - mean[i]) * var[i] * Wv[j]
- if bias:
- v = v + Bv[j]
- xs[i, j] = v
+ if bias:
+ for i, j in T.Parallel(bm, D):
+ xs[i, j] = (x[i, j] - mean[i]) * var[i] * Wv[j] + Bv[j]
+ else:
+ for i, j in T.Parallel(bm, D):
+ xs[i, j] = (x[i, j] - mean[i]) * var[i] * Wv[j]
T.copy(xs, Yb)
T.copy(Yb, Y[bx * bm, 0])
return main
@@ -120,9 +133,10 @@ def add_ln_kernel(D, residual=True, bias=False, eps=1e-5, bm=4, threads=32):
# ----------------------------------------------------------------------------- RoPE (in place on packed qkv)
@tilelang.jit(pass_configs=FAST)
-def rope_kernel(H, Dh, bm=32, threads=128):
+def rope_kernel(H, Dh, bm=32, threads=128, dtype="bfloat16"):
"""QKV[M, 3*H*Dh] packed as (q|k|v)(h)(d). Rotates q and k in place (rotate-half convention, fp32 math).
cos/sin: [L, Dh/2]. Row r has position r % L. M and L are runtime symbols."""
+ DT = _dt(dtype)
M, L = T.dynamic("M"), T.dynamic("L")
half = Dh // 2
W = 2 * H * Dh # q and k columns
@@ -148,13 +162,14 @@ def rope_kernel(H, Dh, bm=32, threads=128):
# ----------------------------------------------------------------------------- flash attention (padding mask + sliding window)
@tilelang.jit(pass_configs=FAST)
-def attn_kernel(B, L, H, Dh, window=0, bm=64, bn=64, stages=1, threads=128):
- """QKV: [B, L, 3, H, Dh] bf16 (a view of the packed [M, 3*H*Dh] buffer). Lens: [B] int32 valid length.
+def attn_kernel(B, L, H, Dh, window=0, bm=64, bn=64, stages=1, threads=128, dtype="bfloat16"):
+ """QKV: [B, L, 3, H, Dh] 16-bit (a view of the packed [M, 3*H*Dh] buffer). Lens: [B] int32 valid length.
O: [B, L, H*Dh]. window>0 => bidirectional sliding window |i-j| <= window. Masked scores use a large
finite negative so fully-masked (padding) rows stay finite.
B and/or L may be None: they then become runtime symbols (one compile serves every shape, at the
cost of predicated loads -- ~4x slower for full attention at L=1024, free for short inputs)."""
+ DT = _dt(dtype)
scale = (1.0 / Dh) ** 0.5 * 1.44269504 # log2(e)
if B is None:
B = T.dynamic("B")
diff --git a/tests/test_doc_tables.py b/tests/test_doc_tables.py
index f57677ca..58a48b4d 100644
--- a/tests/test_doc_tables.py
+++ b/tests/test_doc_tables.py
@@ -72,6 +72,10 @@ HEADER = ("| checkpoint | type | n | max \\|p_fast - p_stock\\| | max \\|p_fast
"| max \\|p_stock - p_fp32\\| | argmax fast = stock | fast = fp32 |")
+FP16_HEADER = ("| checkpoint | type | n | max \\|p_fast - p_fp32\\| bf16 | max \\|p_fast - p_fp32\\| fp16 "
+ "| argmax fast = fp32, bf16 | fp16 |")
+
+
def split_row(line: str) -> List[str]:
return [c.strip() for c in line.strip().strip("|").split("|")]
@@ -154,6 +158,44 @@ def main() -> int:
check("the row that falsified the old sentence is still the multilingual noul one",
contradicted, [("laya-multilingual", "noul")])
+ # ------------------------------------------------- the fp16 table (same files, plus fp16 runs)
+ fp16_sources = {
+ "laya": ("parity_english_rtx4070.json", "parity_english_fp16_rtx4070.json"),
+ "laya-multilingual": ("parity_multilingual_rtx4070.json", "parity_multilingual_fp16_rtx4070.json"),
+ "laya-typed-decisions": ("parity_typed_decisions_rtx4070.json", "parity_typed_decisions_fp16_rtx4070.json"),
+ }
+ pairs: Dict[str, Tuple[Dict[str, Any], Dict[str, Any]]] = {}
+ for ckpt, (bf, fp) in fp16_sources.items():
+ loaded = []
+ for name in (bf, fp):
+ with open(os.path.join(PARITY_DIR, name), encoding="utf-8") as fh:
+ loaded.append(json.load(fh))
+ check("fp16 table/%s bf16 file is a bf16 run" % ckpt, loaded[0]["dtype"], "torch.bfloat16")
+ check("fp16 table/%s fp16 file is an fp16 run" % ckpt, loaded[1]["dtype"], "torch.float16")
+ pairs[ckpt] = (loaded[0]["summary"], loaded[1]["summary"])
+ rows16 = parse_table(read(BENCHMARKS), FP16_HEADER)
+ check_true("fp16 table/parsed nine rows", len(rows16) == 9, "got %d" % len(rows16))
+ for row in rows16:
+ ckpt, qtype = row[0], row[1]
+ if ckpt not in pairs or qtype not in pairs[ckpt][0] or qtype not in pairs[ckpt][1]:
+ FAIL.append("fp16 table/unknown row %s/%s" % (ckpt, qtype))
+ continue
+ b, f = pairs[ckpt][0][qtype], pairs[ckpt][1][qtype]
+ label = "fp16 table/%s/%s" % (ckpt, qtype)
+ check(label + " n", int(row[2]), f["n"])
+ check(label + " bf16 d_fast_fp32", unstyled(row[3]), rounded(b["d_fast_fp32"]))
+ check(label + " fp16 d_fast_fp32", unstyled(row[4]), rounded(f["d_fast_fp32"]))
+ check(label + " bf16 agree_fast_fp32", unstyled(row[5]), "%d/%d" % (b["agree_fast_fp32"], b["n"]))
+ check(label + " fp16 agree_fast_fp32", unstyled(row[6]), "%d/%d" % (f["agree_fast_fp32"], f["n"]))
+ # the prose under it: fp16 fast agrees with fp16 stock everywhere, and moves nothing by more than 0.009
+ fp16_rows = [v for _, f in pairs.values() for v in f.values() if isinstance(v, dict)]
+ check("fp16 prose/argmax fast = stock on every question",
+ sum(v["agree_fast_stock"] for v in fp16_rows), sum(v["n"] for v in fp16_rows))
+ check("fp16 prose/max |fast - stock| rounds to 0.009",
+ rounded(max(v["d_fast_stock"] for v in fp16_rows)), "0.009")
+ check("fp16 prose/README's 'within 0.009 of fp32'",
+ rounded(max(v["d_fast_fp32"] for v in fp16_rows)), "0.009")
+
# ------------------------------------------------- what this cannot check
unbacked = [
("README.md typed-decisions rows", "no committed artifact for the fine-tuned checkpoint"),
diff --git a/tests/test_fast.py b/tests/test_fast.py
index 0e921e1e..5e74ef10 100644
--- a/tests/test_fast.py
+++ b/tests/test_fast.py
@@ -14,46 +14,54 @@ from laya import tl_kernels as K # noqa: E402
dev = "cuda"
def err(a, b): return (a.float() - b.float()).abs().max().item()
+DTYPES = pytest.mark.parametrize("dt", [torch.bfloat16, torch.float16], ids=["bf16", "fp16"])
+def kd(dt): return {torch.bfloat16: "bfloat16", torch.float16: "float16"}[dt]
-def test_gemm_epilogues():
+
+@DTYPES
+def test_gemm_epilogues(dt):
M, Kd = 200, 768 # M not a multiple of the tile on purpose
- A = torch.randn(M, Kd, device=dev, dtype=torch.bfloat16)
+ A = torch.randn(M, Kd, device=dev, dtype=dt)
for N, bias, act in [(2304, False, "none"), (768, True, "gelu"), (3072, True, "relu")]:
- W = torch.randn(N, Kd, device=dev, dtype=torch.bfloat16) * 0.02; b = torch.randn(N, device=dev)
- C = torch.empty(M, N, device=dev, dtype=torch.bfloat16)
- K.gemm_kernel(N, Kd, bias=bias, act=act)(A, W, b, C)
+ W = torch.randn(N, Kd, device=dev, dtype=dt) * 0.02; b = torch.randn(N, device=dev)
+ C = torch.empty(M, N, device=dev, dtype=dt)
+ K.gemm_kernel(N, Kd, bias=bias, act=act, dtype=kd(dt))(A, W, b, C)
ref = A.float() @ W.float().T + (b if bias else 0)
ref = {"none": ref, "gelu": torch.nn.functional.gelu(ref), "relu": torch.relu(ref)}[act]
assert err(C, ref) < 0.05
-def test_geglu():
+@DTYPES
+def test_geglu(dt):
M, Kd, F = 256, 768, 1152
- A = torch.randn(M, Kd, device=dev, dtype=torch.bfloat16); Wi = torch.randn(2 * F, Kd, device=dev, dtype=torch.bfloat16) * 0.02
- C = torch.empty(M, F, device=dev, dtype=torch.bfloat16); K.gemm_geglu_kernel(F, Kd)(A, Wi, C)
+ A = torch.randn(M, Kd, device=dev, dtype=dt); Wi = torch.randn(2 * F, Kd, device=dev, dtype=dt) * 0.02
+ C = torch.empty(M, F, device=dev, dtype=dt); K.gemm_geglu_kernel(F, Kd, dtype=kd(dt))(A, Wi, C)
x = A.float() @ Wi.float().T
assert err(C, torch.nn.functional.gelu(x[:, :F]) * x[:, F:]) < 0.05
-def test_add_layernorm():
+@DTYPES
[email protected]("bias", [True, False])
+def test_add_layernorm(dt, bias):
M, D = 100, 768
X = torch.randn(M, D, device=dev) * 3000 # fp32 residual stream at ModernBERT-large's real magnitude
- R = torch.randn(M, D, device=dev, dtype=torch.bfloat16) * 50
+ R = torch.randn(M, D, device=dev, dtype=dt) * 50
w = torch.rand(D, device=dev) + 0.5; b = torch.randn(D, device=dev)
- X2 = X.clone(); Y = torch.empty(M, D, device=dev, dtype=torch.bfloat16)
- K.add_ln_kernel(D, residual=True, bias=True)(X2, R, w, b, Y)
+ X2 = X.clone(); Y = torch.empty(M, D, device=dev, dtype=dt)
+ K.add_ln_kernel(D, residual=True, bias=bias, dtype=kd(dt))(X2, R, w, b, Y)
xr = X + R.float()
assert torch.equal(X2, xr) # the stream is updated exactly, in fp32
- assert err(Y, torch.nn.functional.layer_norm(xr, (D,), w, b, 1e-5)) < 0.05
+ assert err(Y, torch.nn.functional.layer_norm(xr, (D,), w, b if bias else None, 1e-5)) < 0.05
-def test_attention_mask_and_window():
+@DTYPES
+def test_attention_mask_and_window(dt):
H, Dh, B = 12, 64, 3
for L, window, dyn in [(80, 0, True), (200, 65, True), (1024, 65, False), (1024, 0, False)]:
- qkv = torch.randn(B, L, 3, H, Dh, device=dev, dtype=torch.bfloat16)
+ qkv = torch.randn(B, L, 3, H, Dh, device=dev, dtype=dt)
lens = torch.tensor([L, L - 7, max(1, L // 3)], device=dev, dtype=torch.int32)
- O = torch.empty(B, L, H * Dh, device=dev, dtype=torch.bfloat16)
- K.attn_kernel(None if dyn else B, None if dyn else L, H, Dh, window=window)(qkv, lens, O)
+ O = torch.empty(B, L, H * Dh, device=dev, dtype=dt)
+ K.attn_kernel(None if dyn else B, None if dyn else L, H, Dh, window=window, dtype=kd(dt))(qkv, lens, O)
q, k, v = [qkv[:, :, i].transpose(1, 2).float() for i in range(3)]
idx = torch.arange(L, device=dev)
mask = (idx[None, :] < lens[:, None])[:, None, None, :].expand(B, 1, L, L)
@@ -65,11 +73,18 @@ def test_attention_mask_and_window():
assert (O.float() - ref)[valid].abs().max().item() < 0.02
+def test_kernel_rejects_other_dtypes():
+ with pytest.raises(ValueError):
+ K.gemm_kernel(768, 768, dtype="float32")
+
+
@pytest.mark.skipif(os.environ.get("LAYA_TEST_MODEL") is None, reason="set LAYA_TEST_MODEL=<repo or path> to run")
-def test_full_forward_matches_stock():
+@DTYPES
+def test_full_forward_matches_stock(dt):
import laya
from laya.common import QTYPES, build_sequence, collate_items
agent = laya.load(os.environ["LAYA_TEST_MODEL"], subfolder=os.environ.get("LAYA_TEST_SUBFOLDER"))
+ agent.dtype = dt # the fast path follows the agent's autocast dtype
q = {"dept": {"type": "choice", "instructions": "Which team?", "criteria": {"billing": "refunds", "tech": "bugs", "sales": "pricing"}},
"urg": {"type": "score", "instructions": "How urgent?", "criteria": ["low", "mid", "high"]},
"churn": {"type": "noul", "instructions": "Threatens to cancel?"}}
@@ -83,4 +98,5 @@ def test_full_forward_matches_stock():
with torch.no_grad(), torch.autocast("cuda", dtype=agent.dtype):
return agent.model(b["input_ids"], b["attention_mask"], b["marker_pos"], b["marker_mask"], b["qtype"])[0]
lo = run(); assert agent.accelerate(strict=True); lf = run()
+ assert agent._fast.dtype == dt
assert (torch.softmax(lo.float(), -1) - torch.softmax(lf.float(), -1)).abs().max().item() < 0.02