Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
{
"schema": "laya-option-stability/1",
"reference": {
"base_commit": "ffc142012af63d015fa19f05f29f9c5ff329a2cc",
"overlays": [
{
"commit": "8257e91e4c8c56902720496de75c0a8e2f3f6be7",
"pull_request": "https://github.com/NandhaKishorM/laya/pull/1289"
},
{
"commit": "7feac1d3ba4e30ba09bf658f0ec15001b64a9223",
"pull_request": "https://github.com/NandhaKishorM/laya/pull/1286"
}
],
"method": "git show precision commit source; git apply exact rename commit patch for laya/hooks.py; local combined reference, not upstream main",
"files": {
"laya/hooks.py": "b8f34a77573146b8d6fcdf3d3bc384fac73c52ac612e8fec7218dc3ae9de9321",
"laya/common.py": "8396e8c8dbceabfc7fb2abb1e2fe4692751389ba9bd80d592f1ea922fef0765c"
},
"python": "3.11.13"
},
"variant_cases": [
{"name":"two-dedup","labels":["yes","no"],"seed_key":"","options":{},"expected":[{"name":"original","order":["yes","no"],"relabel":false},{"name":"reversed","order":["no","yes"],"relabel":false},{"name":"letters","order":["yes","no"],"relabel":true},{"name":"letters_reversed","order":["no","yes"],"relabel":true}]},
{"name":"three-utf8","labels":["alpha","beta","gamma"],"seed_key":"种子|café|🙂","options":{},"expected":[{"name":"original","order":["alpha","beta","gamma"],"relabel":false},{"name":"reversed","order":["gamma","beta","alpha"],"relabel":false},{"name":"shuffle2","order":["beta","alpha","gamma"],"relabel":false},{"name":"shuffle3","order":["beta","gamma","alpha"],"relabel":false},{"name":"letters","order":["alpha","beta","gamma"],"relabel":true},{"name":"letters_reversed","order":["gamma","beta","alpha"],"relabel":true},{"name":"letters_shuffle","order":["alpha","gamma","beta"],"relabel":true}]},
{"name":"twenty-zero-shuffles","labels":["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19"],"seed_key":"twenty","options":{"nShuffles":0},"expected":[{"name":"original","order":["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19"],"relabel":false},{"name":"reversed","order":["v19","v18","v17","v16","v15","v14","v13","v12","v11","v10","v9","v8","v7","v6","v5","v4","v3","v2","v1","v0"],"relabel":false},{"name":"letters","order":["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19"],"relabel":true},{"name":"letters_reversed","order":["v19","v18","v17","v16","v15","v14","v13","v12","v11","v10","v9","v8","v7","v6","v5","v4","v3","v2","v1","v0"],"relabel":true},{"name":"letters_shuffle","order":["v14","v5","v1","v11","v8","v6","v17","v12","v9","v7","v0","v3","v4","v15","v10","v16","v2","v13","v18","v19"],"relabel":true}]},
{"name":"twenty-six-twenty-shuffles","labels":["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25"],"seed_key":"twenty-six","options":{"nShuffles":20},"expected":[{"name":"original","order":["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25"],"relabel":false},{"name":"reversed","order":["v25","v24","v23","v22","v21","v20","v19","v18","v17","v16","v15","v14","v13","v12","v11","v10","v9","v8","v7","v6","v5","v4","v3","v2","v1","v0"],"relabel":false},{"name":"shuffle1","order":["v10","v19","v9","v25","v11","v20","v0","v6","v24","v12","v5","v18","v14","v3","v7","v17","v1","v4","v2","v13","v8","v23","v15","v21","v16","v22"],"relabel":false},{"name":"shuffle2","order":["v14","v8","v19","v0","v21","v18","v16","v22","v4","v6","v5","v2","v3","v7","v17","v24","v20","v11","v13","v9","v1","v10","v12","v25","v15","v23"],"relabel":false},{"name":"shuffle3","order":["v8","v18","v16","v23","v7","v5","v17","v22","v3","v19","v12","v9","v6","v4","v11","v0","v1","v15","v25","v20","v10","v14","v13","v24","v21","v2"],"relabel":false},{"name":"shuffle4","order":["v5","v17","v11","v10","v9","v3","v14","v23","v2","v21","v4","v16","v13","v19","v18","v7","v6","v24","v12","v25","v22","v0","v8","v20","v1","v15"],"relabel":false},{"name":"shuffle5","order":["v16","v6","v22","v12","v25","v2","v5","v1","v24","v7","v19","v17","v9","v10","v21","v20","v8","v15","v23","v14","v4","v11","v18","v13","v0","v3"],"relabel":false},{"name":"shuffle6","order":["v7","v16","v24","v11","v1","v15","v22","v14","v20","v21","v5","v2","v0","v6","v3","v8","v10","v13","v23","v18","v4","v19","v25","v9","v12","v17"],"relabel":false},{"name":"shuffle7","order":["v21","v20","v16","v24","v2","v25","v12","v17","v6","v15","v23","v14","v1","v8","v10","v9","v11","v0","v4","v5","v13","v3","v19","v18","v22","v7"],"relabel":false},{"name":"shuffle8","order":["v25","v4","v12","v20","v19","v16","v10","v6","v9","v14","v1","v18","v24","v3","v22","v0","v5","v15","v21","v8","v2","v23","v11","v13","v7","v17"],"relabel":false},{"name":"shuffle9","order":["v3","v2","v1","v0","v11","v20","v24","v12","v17","v8","v18","v7","v16","v5","v21","v14","v22","v6","v10","v19","v15","v13","v25","v4","v9","v23"],"relabel":false},{"name":"shuffle10","order":["v18","v17","v21","v19","v3","v24","v1","v20","v0","v9","v12","v22","v11","v15","v8","v16","v5","v14","v2","v23","v13","v25","v6","v4","v7","v10"],"relabel":false},{"name":"shuffle11","order":["v2","v24","v8","v19","v17","v15","v13","v16","v5","v11","v23","v9","v10","v14","v25","v21","v1","v12","v22","v7","v4","v20","v3","v0","v18","v6"],"relabel":false},{"name":"shuffle12","order":["v8","v12","v13","v11","v24","v6","v25","v22","v0","v23","v9","v1","v2","v5","v4","v19","v17","v20","v21","v3","v7","v14","v18","v16","v15","v10"],"relabel":false},{"name":"shuffle13","order":["v24","v22","v25","v5","v16","v20","v17","v4","v7","v3","v12","v19","v2","v11","v14","v9","v10","v0","v21","v15","v6","v1","v13","v23","v18","v8"],"relabel":false},{"name":"shuffle14","order":["v17","v20","v21","v22","v16","v11","v0","v10","v14","v25","v4","v13","v8","v9","v15","v19","v1","v7","v5","v24","v23","v6","v3","v12","v2","v18"],"relabel":false},{"name":"shuffle15","order":["v13","v4","v19","v10","v2","v11","v7","v1","v6","v3","v24","v23","v18","v9","v8","v16","v17","v20","v21","v15","v5","v14","v12","v22","v0","v25"],"relabel":false},{"name":"shuffle16","order":["v11","v12","v17","v7","v15","v13","v14","v10","v20","v0","v22","v23","v16","v21","v4","v24","v1","v6","v2","v25","v3","v8","v19","v9","v5","v18"],"relabel":false},{"name":"shuffle17","order":["v2","v8","v4","v3","v22","v14","v20","v12","v5","v7","v17","v23","v13","v15","v21","v16","v10","v0","v24","v18","v19","v1","v11","v6","v25","v9"],"relabel":false},{"name":"shuffle18","order":["v6","v1","v14","v25","v2","v22","v3","v20","v8","v23","v18","v0","v4","v7","v13","v17","v21","v5","v11","v19","v24","v16","v12","v15","v10","v9"],"relabel":false},{"name":"shuffle19","order":["v15","v3","v12","v16","v7","v2","v6","v8","v10","v18","v5","v22","v24","v20","v19","v23","v14","v11","v0","v9","v1","v25","v17","v4","v13","v21"],"relabel":false},{"name":"shuffle20","order":["v7","v17","v1","v5","v16","v25","v11","v12","v15","v18","v2","v4","v6","v23","v14","v19","v24","v13","v22","v10","v21","v3","v8","v20","v9","v0"],"relabel":false},{"name":"letters","order":["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25"],"relabel":true},{"name":"letters_reversed","order":["v25","v24","v23","v22","v21","v20","v19","v18","v17","v16","v15","v14","v13","v12","v11","v10","v9","v8","v7","v6","v5","v4","v3","v2","v1","v0"],"relabel":true},{"name":"letters_shuffle","order":["v1","v3","v22","v8","v2","v11","v5","v4","v15","v14","v20","v21","v12","v16","v24","v18","v13","v10","v9","v7","v25","v23","v19","v6","v17","v0"],"relabel":true}]},
{"name":"twenty-seven-no-letters","labels":["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26"],"seed_key":"twenty-seven","options":{},"expected":[{"name":"original","order":["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26"],"relabel":false},{"name":"reversed","order":["v26","v25","v24","v23","v22","v21","v20","v19","v18","v17","v16","v15","v14","v13","v12","v11","v10","v9","v8","v7","v6","v5","v4","v3","v2","v1","v0"],"relabel":false},{"name":"shuffle1","order":["v4","v11","v0","v16","v25","v8","v15","v2","v13","v5","v23","v20","v26","v24","v6","v14","v22","v21","v3","v19","v7","v12","v9","v18","v1","v17","v10"],"relabel":false},{"name":"shuffle2","order":["v1","v4","v23","v20","v8","v6","v12","v14","v9","v7","v13","v16","v0","v21","v17","v25","v11","v10","v2","v19","v15","v24","v22","v26","v18","v5","v3"],"relabel":false},{"name":"shuffle3","order":["v5","v6","v0","v8","v24","v17","v19","v22","v4","v23","v18","v11","v15","v20","v12","v2","v16","v9","v10","v1","v13","v21","v26","v7","v14","v3","v25"],"relabel":false}]},
{"name":"fast-include","labels":["a","b","c"],"seed_key":"seed","options":{"include":["reversed","letters_reversed"]},"expected":[{"name":"original","order":["a","b","c"],"relabel":false},{"name":"reversed","order":["c","b","a"],"relabel":false},{"name":"letters_reversed","order":["c","b","a"],"relabel":true}]},
{"name":"custom-include","labels":["a","b","c","d"],"seed_key":"seed","options":{"include":["shuffle3","letters_shuffle"]},"expected":[{"name":"original","order":["a","b","c","d"],"relabel":false},{"name":"shuffle3","order":["b","d","a","c"],"relabel":false},{"name":"letters_shuffle","order":["a","d","c","b"],"relabel":true}]},
{"name":"rename-only","labels":["a","b","c"],"seed_key":"seed","options":{"reorder":false},"expected":[{"name":"original","order":["a","b","c"],"relabel":false},{"name":"letters","order":["a","b","c"],"relabel":true},{"name":"letters_reversed","order":["c","b","a"],"relabel":true}]},
{"name":"reorder-only","labels":["a","b","c"],"seed_key":"seed","options":{"relabel":false},"expected":[{"name":"original","order":["a","b","c"],"relabel":false},{"name":"reversed","order":["c","b","a"],"relabel":false},{"name":"shuffle2","order":["a","c","b"],"relabel":false}]},
{"name":"no-probes","labels":["a","b","c"],"seed_key":"seed","options":{"relabel":false,"reorder":false},"expected":[{"name":"original","order":["a","b","c"],"relabel":false}]},
{"name":"filtered-away","labels":["a","b","c"],"seed_key":"seed","options":{"nShuffles":0,"include":["shuffle3"]},"expected":[{"name":"original","order":["a","b","c"],"relabel":false}]},
{"name":"unicode-labels","labels":["","🙂","é"],"seed_key":"nul\u0000seed","options":{"nShuffles":5},"expected":[{"name":"original","order":["","🙂","é"],"relabel":false},{"name":"reversed","order":["é","🙂",""],"relabel":false},{"name":"shuffle1","order":["","é","🙂"],"relabel":false},{"name":"shuffle2","order":["é","","🙂"],"relabel":false},{"name":"shuffle5","order":["🙂","","é"],"relabel":false},{"name":"letters","order":["","🙂","é"],"relabel":true},{"name":"letters_reversed","order":["é","🙂",""],"relabel":true},{"name":"letters_shuffle","order":["🙂","é",""],"relabel":true}]}
],
"hook_cases": [
{"name":"described-full","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":"Accept the request","review":"Review the request","reject":"Reject the request"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6},"expanded":[{"key":"q","rendered":["accept: Accept the request","review: Review the request","reject: Reject the request"]},{"key":"q::rel1","rendered":["reject: Reject the request","review: Review the request","accept: Accept the request"]},{"key":"q::rel2","rendered":["review: Review the request","accept: Accept the request","reject: Reject the request"]},{"key":"q::rel3","rendered":["A: Accept the request","B: Review the request","C: Reject the request"]},{"key":"q::rel4","rendered":["A: Reject the request","B: Review the request","C: Accept the request"]},{"key":"q::rel5","rendered":["A: Review the request","B: Accept the request","C: Reject the request"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0}},"q::rel2":{"type":"choice","choice":"accept","probabilities":{"accept":0.7,"review":0.30000000000000004,"reject":0.0}},"q::rel3":{"type":"choice","choice":"A","probabilities":{"A":0.8,"B":0.19999999999999996,"C":0.0}},"q::rel4":{"type":"choice","choice":"B","probabilities":{"C":0.25,"B":0.75,"A":0.0}},"q::rel5":{"type":"choice","choice":"B","probabilities":{"B":0.7,"A":0.30000000000000004,"C":0.0}}},"usage":{"input_tokens":600,"output_tokens":6,"truncated_questions":["q","q::rel1","q::rel2","q::rel3","q::rel4","q::rel5"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3},"q::rel3":{"total":3,"distinct":3},"q::rel4":{"total":3,"distinct":3},"q::rel5":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.6,"rename_stability":0.6667,"reorder_stability":0.5,"n_variants":6,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","reversed":"review","shuffle2":"accept","letters":"accept","letters_reversed":"review","letters_shuffle":"accept"},"variant_support":{"original":0.8,"reversed":0.25,"shuffle2":0.7,"letters":0.8,"letters_reversed":0.25,"letters_shuffle":0.7}}}},"usage":{"input_tokens":600,"output_tokens":6,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"list-fast","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":["accept","review","reject"]}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["accept","review","reject"]},{"key":"q::rel1","rendered":["reject","review","accept"]},{"key":"q::rel2","rendered":["A: reject","B: review","C: accept"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0}},"q::rel2":{"type":"choice","choice":"C","probabilities":{"C":0.7,"B":0.30000000000000004,"A":0.0}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.5,"rename_stability":1.0,"reorder_stability":0.0,"n_variants":3,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","reversed":"review","letters_reversed":"accept"},"variant_support":{"original":0.8,"reversed":0.25,"letters_reversed":0.7}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"undescribed","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":null,"review":"","reject":" \t\n"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["accept","review","reject: \t\n"]},{"key":"q::rel1","rendered":["reject: \t\n","review","accept"]},{"key":"q::rel2","rendered":["A: reject","B: review","C: accept"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0}},"q::rel2":{"type":"choice","choice":"C","probabilities":{"C":0.7,"B":0.30000000000000004,"A":0.0}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.5,"rename_stability":1.0,"reorder_stability":0.0,"n_variants":3,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","reversed":"review","letters_reversed":"accept"},"variant_support":{"original":0.8,"reversed":0.25,"letters_reversed":0.7}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"mixed-values","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"missing":null,"blank":" ","zero":0,"false":false,"object":{"detail":"café","priority":2},"array":["x",3]}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["missing","blank: ","zero: 0","false: false","object: {\"detail\": \"café\", \"priority\": 2}","array: [\"x\", 3]"]},{"key":"q::rel1","rendered":["array: [\"x\", 3]","object: {\"detail\": \"café\", \"priority\": 2}","false: false","zero: 0","blank: ","missing"]},{"key":"q::rel2","rendered":["A: [\"x\", 3]","B: {\"detail\": \"café\", \"priority\": 2}","C: false","D: 0","E: blank","F: missing"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"missing","probabilities":{"missing":0.8,"blank":0.19999999999999996,"zero":0.0,"false":0.0,"object":0.0,"array":0.0}},"q::rel1":{"type":"choice","choice":"blank","probabilities":{"missing":0.25,"blank":0.75,"zero":0.0,"false":0.0,"object":0.0,"array":0.0}},"q::rel2":{"type":"choice","choice":"F","probabilities":{"F":0.7,"E":0.30000000000000004,"D":0.0,"C":0.0,"B":0.0,"A":0.0}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":6,"distinct":6},"q::rel1":{"total":6,"distinct":6},"q::rel2":{"total":6,"distinct":6}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"missing","probabilities":{"missing":0.8,"blank":0.19999999999999996,"zero":0.0,"false":0.0,"object":0.0,"array":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.5,"rename_stability":1.0,"reorder_stability":0.0,"n_variants":3,"distinct_choices":["blank","missing"],"variant_choices":{"original":"missing","reversed":"blank","letters_reversed":"missing"},"variant_support":{"original":0.8,"reversed":0.25,"letters_reversed":0.7}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":6,"distinct":6}}}}]},
{"name":"numeric-keys-order","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"1":"first","2":"second","10":"tenth"},"option_order":[2,0,1]}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6},"expanded":[{"key":"q","rendered":["10: tenth","1: first","2: second"]},{"key":"q::rel1","rendered":["1: first","10: tenth","2: second"]},{"key":"q::rel2","rendered":["10: tenth","2: second","1: first"]},{"key":"q::rel3","rendered":["C: tenth","A: first","B: second"]},{"key":"q::rel4","rendered":["C: first","A: tenth","B: second"]},{"key":"q::rel5","rendered":["C: tenth","A: second","B: first"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"1","probabilities":{"1":0.8,"2":0.19999999999999996,"10":0.0}},"q::rel1":{"type":"choice","choice":"2","probabilities":{"1":0.25,"2":0.75,"10":0.0}},"q::rel2":{"type":"choice","choice":"1","probabilities":{"1":0.7,"2":0.30000000000000004,"10":0.0}},"q::rel3":{"type":"choice","choice":"A","probabilities":{"A":0.8,"B":0.19999999999999996,"C":0.0}},"q::rel4":{"type":"choice","choice":"B","probabilities":{"C":0.25,"B":0.75,"A":0.0}},"q::rel5":{"type":"choice","choice":"B","probabilities":{"B":0.7,"A":0.30000000000000004,"C":0.0}}},"usage":{"input_tokens":600,"output_tokens":6,"truncated_questions":["q","q::rel1","q::rel2","q::rel3","q::rel4","q::rel5"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3},"q::rel3":{"total":3,"distinct":3},"q::rel4":{"total":3,"distinct":3},"q::rel5":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"1","probabilities":{"1":0.8,"2":0.19999999999999996,"10":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.6,"rename_stability":0.6667,"reorder_stability":0.5,"n_variants":6,"distinct_choices":["1","2"],"variant_choices":{"original":"1","reversed":"2","shuffle2":"1","letters":"1","letters_reversed":"2","letters_shuffle":"1"},"variant_support":{"original":0.8,"reversed":0.25,"shuffle2":0.7,"letters":0.8,"letters_reversed":0.25,"letters_shuffle":0.7}}}},"usage":{"input_tokens":600,"output_tokens":6,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"numeric-list","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":["1","2","10"]}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["1","2","10"]},{"key":"q::rel1","rendered":["10","2","1"]},{"key":"q::rel2","rendered":["A: 10","B: 2","C: 1"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"1","probabilities":{"1":0.8,"2":0.19999999999999996,"10":0.0}},"q::rel1":{"type":"choice","choice":"2","probabilities":{"1":0.25,"2":0.75,"10":0.0}},"q::rel2":{"type":"choice","choice":"C","probabilities":{"C":0.7,"B":0.30000000000000004,"A":0.0}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"1","probabilities":{"1":0.8,"2":0.19999999999999996,"10":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.5,"rename_stability":1.0,"reorder_stability":0.0,"n_variants":3,"distinct_choices":["1","2"],"variant_choices":{"original":"1","reversed":"2","letters_reversed":"1"},"variant_support":{"original":0.8,"reversed":0.25,"letters_reversed":0.7}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"prototype-labels","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"__proto__":"base","constructor":"create","toString":"print"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6},"expanded":[{"key":"q","rendered":["__proto__: base","constructor: create","toString: print"]},{"key":"q::rel1","rendered":["toString: print","constructor: create","__proto__: base"]},{"key":"q::rel2","rendered":["constructor: create","__proto__: base","toString: print"]},{"key":"q::rel3","rendered":["A: base","B: create","C: print"]},{"key":"q::rel4","rendered":["A: print","B: create","C: base"]},{"key":"q::rel5","rendered":["A: create","B: base","C: print"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"__proto__","probabilities":{"__proto__":0.8,"constructor":0.19999999999999996,"toString":0.0}},"q::rel1":{"type":"choice","choice":"constructor","probabilities":{"__proto__":0.25,"constructor":0.75,"toString":0.0}},"q::rel2":{"type":"choice","choice":"__proto__","probabilities":{"__proto__":0.7,"constructor":0.30000000000000004,"toString":0.0}},"q::rel3":{"type":"choice","choice":"A","probabilities":{"A":0.8,"B":0.19999999999999996,"C":0.0}},"q::rel4":{"type":"choice","choice":"B","probabilities":{"C":0.25,"B":0.75,"A":0.0}},"q::rel5":{"type":"choice","choice":"B","probabilities":{"B":0.7,"A":0.30000000000000004,"C":0.0}}},"usage":{"input_tokens":600,"output_tokens":6,"truncated_questions":["q","q::rel1","q::rel2","q::rel3","q::rel4","q::rel5"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3},"q::rel3":{"total":3,"distinct":3},"q::rel4":{"total":3,"distinct":3},"q::rel5":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"__proto__","probabilities":{"__proto__":0.8,"constructor":0.19999999999999996,"toString":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.6,"rename_stability":0.6667,"reorder_stability":0.5,"n_variants":6,"distinct_choices":["__proto__","constructor"],"variant_choices":{"original":"__proto__","reversed":"constructor","shuffle2":"__proto__","letters":"__proto__","letters_reversed":"constructor","letters_shuffle":"__proto__"},"variant_support":{"original":0.8,"reversed":0.25,"shuffle2":0.7,"letters":0.8,"letters_reversed":0.25,"letters_shuffle":0.7}}}},"usage":{"input_tokens":600,"output_tokens":6,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"unicode-labels","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"":"private","🙂":"smile","é":"accent"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6},"expanded":[{"key":"q","rendered":[": private","🙂: smile","é: accent"]},{"key":"q::rel1","rendered":["é: accent","🙂: smile",": private"]},{"key":"q::rel2","rendered":["🙂: smile",": private","é: accent"]},{"key":"q::rel3","rendered":["A: private","B: smile","C: accent"]},{"key":"q::rel4","rendered":["A: accent","B: smile","C: private"]},{"key":"q::rel5","rendered":["A: smile","B: private","C: accent"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"","probabilities":{"":0.8,"🙂":0.19999999999999996,"é":0.0}},"q::rel1":{"type":"choice","choice":"🙂","probabilities":{"":0.25,"🙂":0.75,"é":0.0}},"q::rel2":{"type":"choice","choice":"","probabilities":{"":0.7,"🙂":0.30000000000000004,"é":0.0}},"q::rel3":{"type":"choice","choice":"A","probabilities":{"A":0.8,"B":0.19999999999999996,"C":0.0}},"q::rel4":{"type":"choice","choice":"B","probabilities":{"C":0.25,"B":0.75,"A":0.0}},"q::rel5":{"type":"choice","choice":"B","probabilities":{"B":0.7,"A":0.30000000000000004,"C":0.0}}},"usage":{"input_tokens":600,"output_tokens":6,"truncated_questions":["q","q::rel1","q::rel2","q::rel3","q::rel4","q::rel5"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3},"q::rel3":{"total":3,"distinct":3},"q::rel4":{"total":3,"distinct":3},"q::rel5":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"","probabilities":{"":0.8,"🙂":0.19999999999999996,"é":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.6,"rename_stability":0.6667,"reorder_stability":0.5,"n_variants":6,"distinct_choices":["","🙂"],"variant_choices":{"original":"","reversed":"🙂","shuffle2":"","letters":"","letters_reversed":"🙂","letters_shuffle":""},"variant_support":{"original":0.8,"reversed":0.25,"shuffle2":0.7,"letters":0.8,"letters_reversed":0.25,"letters_shuffle":0.7}}}},"usage":{"input_tokens":600,"output_tokens":6,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"custom-variants","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":"Accept the request","review":"Review the request","reject":"Reject the request"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":["shuffle3","letters_shuffle"],"seed":"种子🙂"},"expanded":[{"key":"q","rendered":["accept: Accept the request","review: Review the request","reject: Reject the request"]},{"key":"q::rel1","rendered":["A: Review the request","B: Accept the request","C: Reject the request"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"A","probabilities":{"B":0.25,"A":0.75,"C":0.0}}},"usage":{"input_tokens":200,"output_tokens":2,"truncated_questions":["q","q::rel1"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.525,"stability":0.0,"rename_stability":0.0,"reorder_stability":null,"n_variants":2,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","letters_shuffle":"review"},"variant_support":{"original":0.8,"letters_shuffle":0.25}}}},"usage":{"input_tokens":200,"output_tokens":2,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"zero-shuffles","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":"Accept the request","review":"Review the request","reject":"Reject the request"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"nShuffles":0},"expanded":[{"key":"q","rendered":["accept: Accept the request","review: Review the request","reject: Reject the request"]},{"key":"q::rel1","rendered":["reject: Reject the request","review: Review the request","accept: Accept the request"]},{"key":"q::rel2","rendered":["A: Accept the request","B: Review the request","C: Reject the request"]},{"key":"q::rel3","rendered":["A: Reject the request","B: Review the request","C: Accept the request"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0}},"q::rel2":{"type":"choice","choice":"A","probabilities":{"A":0.7,"B":0.30000000000000004,"C":0.0}},"q::rel3":{"type":"choice","choice":"C","probabilities":{"C":0.8,"B":0.19999999999999996,"A":0.0}}},"usage":{"input_tokens":400,"output_tokens":4,"truncated_questions":["q","q::rel1","q::rel2","q::rel3"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3},"q::rel3":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"VERIFY","soft_stability":0.6375,"stability":0.6667,"rename_stability":1.0,"reorder_stability":0.0,"n_variants":4,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","reversed":"review","letters":"accept","letters_reversed":"accept"},"variant_support":{"original":0.8,"reversed":0.25,"letters":0.7,"letters_reversed":0.8}}}},"usage":{"input_tokens":400,"output_tokens":4,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"extra-shuffles","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":"Accept the request","review":"Review the request","reject":"Reject the request"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"nShuffles":5},"expanded":[{"key":"q","rendered":["accept: Accept the request","review: Review the request","reject: Reject the request"]},{"key":"q::rel1","rendered":["reject: Reject the request","review: Review the request","accept: Accept the request"]},{"key":"q::rel2","rendered":["review: Review the request","accept: Accept the request","reject: Reject the request"]},{"key":"q::rel3","rendered":["review: Review the request","reject: Reject the request","accept: Accept the request"]},{"key":"q::rel4","rendered":["A: Accept the request","B: Review the request","C: Reject the request"]},{"key":"q::rel5","rendered":["A: Reject the request","B: Review the request","C: Accept the request"]},{"key":"q::rel6","rendered":["A: Review the request","B: Accept the request","C: Reject the request"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0}},"q::rel2":{"type":"choice","choice":"accept","probabilities":{"accept":0.7,"review":0.30000000000000004,"reject":0.0}},"q::rel3":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel4":{"type":"choice","choice":"B","probabilities":{"A":0.25,"B":0.75,"C":0.0}},"q::rel5":{"type":"choice","choice":"C","probabilities":{"C":0.7,"B":0.30000000000000004,"A":0.0}},"q::rel6":{"type":"choice","choice":"B","probabilities":{"B":0.8,"A":0.19999999999999996,"C":0.0}}},"usage":{"input_tokens":700,"output_tokens":7,"truncated_questions":["q","q::rel1","q::rel2","q::rel3","q::rel4","q::rel5","q::rel6"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3},"q::rel3":{"total":3,"distinct":3},"q::rel4":{"total":3,"distinct":3},"q::rel5":{"total":3,"distinct":3},"q::rel6":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"VERIFY","soft_stability":0.6142857142857142,"stability":0.6667,"rename_stability":0.6667,"reorder_stability":0.6667,"n_variants":7,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","reversed":"review","shuffle2":"accept","shuffle5":"accept","letters":"review","letters_reversed":"accept","letters_shuffle":"accept"},"variant_support":{"original":0.8,"reversed":0.25,"shuffle2":0.7,"shuffle5":0.8,"letters":0.25,"letters_reversed":0.7,"letters_shuffle":0.8}}}},"usage":{"input_tokens":700,"output_tokens":7,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"uncalibrated","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":"Accept the request","review":"Review the request","reject":"Reject the request"}}},"layout":"sequential","options":{"accept":null,"escalate":null,"variants":"fast"},"expanded":[{"key":"q","rendered":["accept: Accept the request","review: Review the request","reject: Reject the request"]},{"key":"q::rel1","rendered":["reject: Reject the request","review: Review the request","accept: Accept the request"]},{"key":"q::rel2","rendered":["A: Reject the request","B: Review the request","C: Accept the request"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0}},"q::rel2":{"type":"choice","choice":"C","probabilities":{"C":0.7,"B":0.30000000000000004,"A":0.0}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":null,"soft_stability":0.5833333333333334,"stability":0.5,"rename_stability":1.0,"reorder_stability":0.0,"n_variants":3,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","reversed":"review","letters_reversed":"accept"},"variant_support":{"original":0.8,"reversed":0.25,"letters_reversed":0.7}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"parallel-auto","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":"Accept the request","review":"Review the request","reject":"Reject the request"}}},"layout":"parallel","options":{"accept":0.8,"escalate":0.6},"expanded":[{"key":"q","rendered":["accept: Accept the request","review: Review the request","reject: Reject the request"]},{"key":"q::rel1","rendered":["A: Accept the request","B: Review the request","C: Reject the request"]},{"key":"q::rel2","rendered":["A: Reject the request","B: Review the request","C: Accept the request"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"B","probabilities":{"A":0.25,"B":0.75,"C":0.0}},"q::rel2":{"type":"choice","choice":"C","probabilities":{"C":0.7,"B":0.30000000000000004,"A":0.0}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["rename"],"option_layout":"parallel","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.5,"rename_stability":0.5,"reorder_stability":null,"n_variants":3,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","letters":"review","letters_reversed":"accept"},"variant_support":{"original":0.8,"letters":0.25,"letters_reversed":0.7}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"parallel-explicit-reorder","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":"Accept the request","review":"Review the request","reject":"Reject the request"}}},"layout":"parallel","options":{"accept":0.8,"escalate":0.6,"probes":"reorder"},"expanded":[{"key":"q","rendered":["accept: Accept the request","review: Review the request","reject: Reject the request"]},{"key":"q::rel1","rendered":["reject: Reject the request","review: Review the request","accept: Accept the request"]},{"key":"q::rel2","rendered":["review: Review the request","accept: Accept the request","reject: Reject the request"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0}},"q::rel2":{"type":"choice","choice":"accept","probabilities":{"accept":0.7,"review":0.30000000000000004,"reject":0.0}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["reorder"],"option_layout":"parallel","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.5,"rename_stability":null,"reorder_stability":0.5,"n_variants":3,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","reversed":"review","shuffle2":"accept"},"variant_support":{"original":0.8,"reversed":0.25,"shuffle2":0.7}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"parallel-relabel-off","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":"Accept the request","review":"Review the request","reject":"Reject the request"}}},"layout":"parallel","options":{"accept":0.8,"escalate":0.6,"relabel":false},"expanded":[{"key":"q","rendered":["accept: Accept the request","review: Review the request","reject: Reject the request"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}}},"usage":{"input_tokens":100,"output_tokens":1,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":[],"option_layout":"parallel","decision":null,"soft_stability":null,"stability":null,"rename_stability":null,"reorder_stability":null,"n_variants":1,"distinct_choices":["accept"],"variant_choices":{"original":"accept"},"variant_support":{"original":0.8}}}},"usage":{"input_tokens":100,"output_tokens":1,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"rename-cap","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"v0":"description 0","v1":"description 1","v2":"description 2","v3":"description 3","v4":"description 4","v5":"description 5","v6":"description 6","v7":"description 7","v8":"description 8","v9":"description 9","v10":"description 10","v11":"description 11","v12":"description 12","v13":"description 13","v14":"description 14","v15":"description 15","v16":"description 16","v17":"description 17","v18":"description 18","v19":"description 19"}}},"layout":"parallel","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["v0: description 0","v1: description 1","v2: description 2","v3: description 3","v4: description 4","v5: description 5","v6: description 6","v7: description 7","v8: description 8","v9: description 9","v10: description 10","v11: description 11","v12: description 12","v13: description 13","v14: description 14","v15: description 15","v16: description 16","v17: description 17","v18: description 18","v19: description 19"]},{"key":"q::rel1","rendered":["A: description 19","B: description 18","C: description 17","D: description 16","E: description 15","F: description 14","G: description 13","H: description 12","I: description 11","J: description 10","K: description 9","L: description 8","M: description 7","N: description 6","O: description 5","P: description 4","Q: description 3","R: description 2","S: description 1","T: description 0"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"v0","probabilities":{"v0":0.8,"v1":0.19999999999999996,"v2":0.0,"v3":0.0,"v4":0.0,"v5":0.0,"v6":0.0,"v7":0.0,"v8":0.0,"v9":0.0,"v10":0.0,"v11":0.0,"v12":0.0,"v13":0.0,"v14":0.0,"v15":0.0,"v16":0.0,"v17":0.0,"v18":0.0,"v19":0.0}},"q::rel1":{"type":"choice","choice":"S","probabilities":{"T":0.25,"S":0.75,"R":0.0,"Q":0.0,"P":0.0,"O":0.0,"N":0.0,"M":0.0,"L":0.0,"K":0.0,"J":0.0,"I":0.0,"H":0.0,"G":0.0,"F":0.0,"E":0.0,"D":0.0,"C":0.0,"B":0.0,"A":0.0}}},"usage":{"input_tokens":200,"output_tokens":2,"truncated_questions":["q","q::rel1"],"options":{"q":{"total":20,"distinct":20},"q::rel1":{"total":20,"distinct":20}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"v0","probabilities":{"v0":0.8,"v1":0.19999999999999996,"v2":0.0,"v3":0.0,"v4":0.0,"v5":0.0,"v6":0.0,"v7":0.0,"v8":0.0,"v9":0.0,"v10":0.0,"v11":0.0,"v12":0.0,"v13":0.0,"v14":0.0,"v15":0.0,"v16":0.0,"v17":0.0,"v18":0.0,"v19":0.0},"reliability":{"probes":["rename"],"option_layout":"parallel","decision":"ESCALATE","soft_stability":0.525,"stability":0.0,"rename_stability":0.0,"reorder_stability":null,"n_variants":2,"distinct_choices":["v0","v1"],"variant_choices":{"original":"v0","letters_reversed":"v1"},"variant_support":{"original":0.8,"letters_reversed":0.25}}}},"usage":{"input_tokens":200,"output_tokens":2,"truncated_questions":["q"],"options":{"q":{"total":20,"distinct":20}}}}]},
{"name":"over-rename-cap","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"v0":null,"v1":null,"v2":null,"v3":null,"v4":null,"v5":null,"v6":null,"v7":null,"v8":null,"v9":null,"v10":null,"v11":null,"v12":null,"v13":null,"v14":null,"v15":null,"v16":null,"v17":null,"v18":null,"v19":null,"v20":null}}},"layout":"parallel","options":{"accept":0.8,"escalate":0.6},"expanded":[{"key":"q","rendered":["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19","v20"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"v0","probabilities":{"v0":0.8,"v1":0.19999999999999996,"v2":0.0,"v3":0.0,"v4":0.0,"v5":0.0,"v6":0.0,"v7":0.0,"v8":0.0,"v9":0.0,"v10":0.0,"v11":0.0,"v12":0.0,"v13":0.0,"v14":0.0,"v15":0.0,"v16":0.0,"v17":0.0,"v18":0.0,"v19":0.0,"v20":0.0}}},"usage":{"input_tokens":100,"output_tokens":1,"truncated_questions":["q"],"options":{"q":{"total":21,"distinct":21}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"v0","probabilities":{"v0":0.8,"v1":0.19999999999999996,"v2":0.0,"v3":0.0,"v4":0.0,"v5":0.0,"v6":0.0,"v7":0.0,"v8":0.0,"v9":0.0,"v10":0.0,"v11":0.0,"v12":0.0,"v13":0.0,"v14":0.0,"v15":0.0,"v16":0.0,"v17":0.0,"v18":0.0,"v19":0.0,"v20":0.0},"reliability":{"probes":[],"option_layout":"parallel","decision":null,"soft_stability":null,"stability":null,"rename_stability":null,"reorder_stability":null,"n_variants":1,"distinct_choices":["v0"],"variant_choices":{"original":"v0"},"variant_support":{"original":0.8}}}},"usage":{"input_tokens":100,"output_tokens":1,"truncated_questions":["q"],"options":{"q":{"total":21,"distinct":21}}}}]},
{"name":"alphabet-cap","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"v0":null,"v1":null,"v2":null,"v3":null,"v4":null,"v5":null,"v6":null,"v7":null,"v8":null,"v9":null,"v10":null,"v11":null,"v12":null,"v13":null,"v14":null,"v15":null,"v16":null,"v17":null,"v18":null,"v19":null,"v20":null,"v21":null,"v22":null,"v23":null,"v24":null,"v25":null,"v26":null}}},"layout":"parallel","options":{"accept":0.8,"escalate":0.6,"renameMaxOptions":27},"expanded":[{"key":"q","rendered":["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19","v20","v21","v22","v23","v24","v25","v26"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"v0","probabilities":{"v0":0.8,"v1":0.19999999999999996,"v2":0.0,"v3":0.0,"v4":0.0,"v5":0.0,"v6":0.0,"v7":0.0,"v8":0.0,"v9":0.0,"v10":0.0,"v11":0.0,"v12":0.0,"v13":0.0,"v14":0.0,"v15":0.0,"v16":0.0,"v17":0.0,"v18":0.0,"v19":0.0,"v20":0.0,"v21":0.0,"v22":0.0,"v23":0.0,"v24":0.0,"v25":0.0,"v26":0.0}}},"usage":{"input_tokens":100,"output_tokens":1,"truncated_questions":["q"],"options":{"q":{"total":27,"distinct":27}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"v0","probabilities":{"v0":0.8,"v1":0.19999999999999996,"v2":0.0,"v3":0.0,"v4":0.0,"v5":0.0,"v6":0.0,"v7":0.0,"v8":0.0,"v9":0.0,"v10":0.0,"v11":0.0,"v12":0.0,"v13":0.0,"v14":0.0,"v15":0.0,"v16":0.0,"v17":0.0,"v18":0.0,"v19":0.0,"v20":0.0,"v21":0.0,"v22":0.0,"v23":0.0,"v24":0.0,"v25":0.0,"v26":0.0},"reliability":{"probes":[],"option_layout":"parallel","decision":null,"soft_stability":null,"stability":null,"rename_stability":null,"reorder_stability":null,"n_variants":1,"distinct_choices":["v0"],"variant_choices":{"original":"v0"},"variant_support":{"original":0.8}}}},"usage":{"input_tokens":100,"output_tokens":1,"truncated_questions":["q"],"options":{"q":{"total":27,"distinct":27}}}}]},
{"name":"one-option","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"only":null}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6},"expanded":[{"key":"q","rendered":["only"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"only","probabilities":{"only":1.0}}},"usage":{"input_tokens":100,"output_tokens":1,"truncated_questions":["q"],"options":{"q":{"total":1,"distinct":1}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"only","probabilities":{"only":1.0},"reliability":{"probes":[],"option_layout":"sequential","decision":null,"soft_stability":null,"stability":null,"rename_stability":null,"reorder_stability":null,"n_variants":1,"distinct_choices":["only"],"variant_choices":{"original":"only"},"variant_support":{"original":1.0}}}},"usage":{"input_tokens":100,"output_tokens":1,"truncated_questions":["q"],"options":{"q":{"total":1,"distinct":1}}}}]},
{"name":"batch-seed","states":["first","second"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":"Accept the request","review":"Review the request","reject":"Reject the request"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["accept: Accept the request","review: Review the request","reject: Reject the request"]},{"key":"q::rel1","rendered":["reject: Reject the request","review: Review the request","accept: Accept the request"]},{"key":"q::rel2","rendered":["A: Reject the request","B: Review the request","C: Accept the request"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0}},"q::rel2":{"type":"choice","choice":"C","probabilities":{"C":0.7,"B":0.30000000000000004,"A":0.0}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3}}}},{"answers":{"q":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0}},"q::rel1":{"type":"choice","choice":"accept","probabilities":{"accept":0.7,"review":0.30000000000000004,"reject":0.0}},"q::rel2":{"type":"choice","choice":"C","probabilities":{"C":0.8,"B":0.19999999999999996,"A":0.0}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.5,"rename_stability":1.0,"reorder_stability":0.0,"n_variants":3,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","reversed":"review","letters_reversed":"accept"},"variant_support":{"original":0.8,"reversed":0.25,"letters_reversed":0.7}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}},{"answers":{"q":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.4166666666666667,"stability":0.0,"rename_stability":0.0,"reorder_stability":0.0,"n_variants":3,"distinct_choices":["accept","review"],"variant_choices":{"original":"review","reversed":"accept","letters_reversed":"accept"},"variant_support":{"original":0.75,"reversed":0.3,"letters_reversed":0.2}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"explicit-structured-state","states":[{"user":"你好","enabled":true}],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"accept":"Accept the request","review":"Review the request","reject":"Reject the request"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"seed":"shared seed"},"expanded":[{"key":"q","rendered":["accept: Accept the request","review: Review the request","reject: Reject the request"]},{"key":"q::rel1","rendered":["reject: Reject the request","review: Review the request","accept: Accept the request"]},{"key":"q::rel2","rendered":["reject: Reject the request","accept: Accept the request","review: Review the request"]},{"key":"q::rel3","rendered":["accept: Accept the request","reject: Reject the request","review: Review the request"]},{"key":"q::rel4","rendered":["A: Accept the request","B: Review the request","C: Reject the request"]},{"key":"q::rel5","rendered":["A: Reject the request","B: Review the request","C: Accept the request"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel1":{"type":"choice","choice":"review","probabilities":{"accept":0.25,"review":0.75,"reject":0.0}},"q::rel2":{"type":"choice","choice":"accept","probabilities":{"accept":0.7,"review":0.30000000000000004,"reject":0.0}},"q::rel3":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0}},"q::rel4":{"type":"choice","choice":"B","probabilities":{"A":0.25,"B":0.75,"C":0.0}},"q::rel5":{"type":"choice","choice":"C","probabilities":{"C":0.7,"B":0.30000000000000004,"A":0.0}}},"usage":{"input_tokens":600,"output_tokens":6,"truncated_questions":["q","q::rel1","q::rel2","q::rel3","q::rel4","q::rel5"],"options":{"q":{"total":3,"distinct":3},"q::rel1":{"total":3,"distinct":3},"q::rel2":{"total":3,"distinct":3},"q::rel3":{"total":3,"distinct":3},"q::rel4":{"total":3,"distinct":3},"q::rel5":{"total":3,"distinct":3}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"accept","probabilities":{"accept":0.8,"review":0.19999999999999996,"reject":0.0},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5833333333333334,"stability":0.6,"rename_stability":0.5,"reorder_stability":0.6667,"n_variants":6,"distinct_choices":["accept","review"],"variant_choices":{"original":"accept","reversed":"review","shuffle1":"accept","shuffle2":"accept","letters":"review","letters_reversed":"accept"},"variant_support":{"original":0.8,"reversed":0.25,"shuffle1":0.7,"shuffle2":0.8,"letters":0.25,"letters_reversed":0.7}}}},"usage":{"input_tokens":600,"output_tokens":6,"truncated_questions":["q"],"options":{"q":{"total":3,"distinct":3}}}}]},
{"name":"rounding-exact-halves","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"yes":"allow","no":"deny"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["yes: allow","no: deny"]},{"key":"q::rel1","rendered":["no: deny","yes: allow"]},{"key":"q::rel2","rendered":["A: deny","B: allow"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.9,"no":0.09999999999999998}},"q::rel1":{"type":"choice","choice":"no","probabilities":{"yes":0.03125,"no":0.96875}},"q::rel2":{"type":"choice","choice":"A","probabilities":{"B":0.15625,"A":0.84375}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":2,"distinct":2},"q::rel1":{"total":2,"distinct":2},"q::rel2":{"total":2,"distinct":2}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.9,"no":0.09999999999999998},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.3625,"stability":0.0,"rename_stability":0.0,"reorder_stability":0.0,"n_variants":3,"distinct_choices":["no","yes"],"variant_choices":{"original":"yes","reversed":"no","letters_reversed":"no"},"variant_support":{"original":0.9,"reversed":0.0312,"letters_reversed":0.1562}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":2,"distinct":2}}}}]},
{"name":"rounding-binary-neighbors","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"yes":"allow","no":"deny"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["yes: allow","no: deny"]},{"key":"q::rel1","rendered":["no: deny","yes: allow"]},{"key":"q::rel2","rendered":["A: deny","B: allow"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.9,"no":0.09999999999999998}},"q::rel1":{"type":"choice","choice":"no","probabilities":{"yes":0.12345,"no":0.8765499999999999}},"q::rel2":{"type":"choice","choice":"B","probabilities":{"B":0.99995,"A":4.999999999999449e-05}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":2,"distinct":2},"q::rel1":{"total":2,"distinct":2},"q::rel2":{"total":2,"distinct":2}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.9,"no":0.09999999999999998},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"VERIFY","soft_stability":0.6744666666666667,"stability":0.5,"rename_stability":1.0,"reorder_stability":0.0,"n_variants":3,"distinct_choices":["no","yes"],"variant_choices":{"original":"yes","reversed":"no","letters_reversed":"yes"},"variant_support":{"original":0.9,"reversed":0.1235,"letters_reversed":1.0}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":2,"distinct":2}}}}]},
{"name":"below-accept","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"yes":"allow","no":"deny"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["yes: allow","no: deny"]},{"key":"q::rel1","rendered":["no: deny","yes: allow"]},{"key":"q::rel2","rendered":["A: deny","B: allow"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.7999,"no":0.20009999999999994}},"q::rel1":{"type":"choice","choice":"yes","probabilities":{"yes":0.8,"no":0.19999999999999996}},"q::rel2":{"type":"choice","choice":"B","probabilities":{"B":0.8,"A":0.19999999999999996}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":2,"distinct":2},"q::rel1":{"total":2,"distinct":2},"q::rel2":{"total":2,"distinct":2}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.7999,"no":0.20009999999999994},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"VERIFY","soft_stability":0.7999666666666667,"stability":1.0,"rename_stability":1.0,"reorder_stability":1.0,"n_variants":3,"distinct_choices":["yes"],"variant_choices":{"original":"yes","reversed":"yes","letters_reversed":"yes"},"variant_support":{"original":0.7999,"reversed":0.8,"letters_reversed":0.8}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":2,"distinct":2}}}}]},
{"name":"at-accept","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"yes":"allow","no":"deny"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["yes: allow","no: deny"]},{"key":"q::rel1","rendered":["no: deny","yes: allow"]},{"key":"q::rel2","rendered":["A: deny","B: allow"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.8,"no":0.19999999999999996}},"q::rel1":{"type":"choice","choice":"yes","probabilities":{"yes":0.8,"no":0.19999999999999996}},"q::rel2":{"type":"choice","choice":"B","probabilities":{"B":0.8,"A":0.19999999999999996}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":2,"distinct":2},"q::rel1":{"total":2,"distinct":2},"q::rel2":{"total":2,"distinct":2}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.8,"no":0.19999999999999996},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ACCEPT","soft_stability":0.8000000000000002,"stability":1.0,"rename_stability":1.0,"reorder_stability":1.0,"n_variants":3,"distinct_choices":["yes"],"variant_choices":{"original":"yes","reversed":"yes","letters_reversed":"yes"},"variant_support":{"original":0.8,"reversed":0.8,"letters_reversed":0.8}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":2,"distinct":2}}}}]},
{"name":"above-accept","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"yes":"allow","no":"deny"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["yes: allow","no: deny"]},{"key":"q::rel1","rendered":["no: deny","yes: allow"]},{"key":"q::rel2","rendered":["A: deny","B: allow"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.8001,"no":0.19989999999999997}},"q::rel1":{"type":"choice","choice":"yes","probabilities":{"yes":0.8,"no":0.19999999999999996}},"q::rel2":{"type":"choice","choice":"B","probabilities":{"B":0.8,"A":0.19999999999999996}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":2,"distinct":2},"q::rel1":{"total":2,"distinct":2},"q::rel2":{"total":2,"distinct":2}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.8001,"no":0.19989999999999997},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ACCEPT","soft_stability":0.8000333333333334,"stability":1.0,"rename_stability":1.0,"reorder_stability":1.0,"n_variants":3,"distinct_choices":["yes"],"variant_choices":{"original":"yes","reversed":"yes","letters_reversed":"yes"},"variant_support":{"original":0.8001,"reversed":0.8,"letters_reversed":0.8}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":2,"distinct":2}}}}]},
{"name":"below-escalate","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"yes":"allow","no":"deny"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["yes: allow","no: deny"]},{"key":"q::rel1","rendered":["no: deny","yes: allow"]},{"key":"q::rel2","rendered":["A: deny","B: allow"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.5999,"no":0.4001}},"q::rel1":{"type":"choice","choice":"yes","probabilities":{"yes":0.6,"no":0.4}},"q::rel2":{"type":"choice","choice":"B","probabilities":{"B":0.6,"A":0.4}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":2,"distinct":2},"q::rel1":{"total":2,"distinct":2},"q::rel2":{"total":2,"distinct":2}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.5999,"no":0.4001},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"ESCALATE","soft_stability":0.5999666666666666,"stability":1.0,"rename_stability":1.0,"reorder_stability":1.0,"n_variants":3,"distinct_choices":["yes"],"variant_choices":{"original":"yes","reversed":"yes","letters_reversed":"yes"},"variant_support":{"original":0.5999,"reversed":0.6,"letters_reversed":0.6}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":2,"distinct":2}}}}]},
{"name":"at-escalate","states":["fixture state"],"questions":{"q":{"type":"choice","instructions":"Choose a suitable action.","criteria":{"yes":"allow","no":"deny"}}},"layout":"sequential","options":{"accept":0.8,"escalate":0.6,"variants":"fast"},"expanded":[{"key":"q","rendered":["yes: allow","no: deny"]},{"key":"q::rel1","rendered":["no: deny","yes: allow"]},{"key":"q::rel2","rendered":["A: deny","B: allow"]}],"model_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.6,"no":0.4}},"q::rel1":{"type":"choice","choice":"yes","probabilities":{"yes":0.6,"no":0.4}},"q::rel2":{"type":"choice","choice":"B","probabilities":{"B":0.6,"A":0.4}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q","q::rel1","q::rel2"],"options":{"q":{"total":2,"distinct":2},"q::rel1":{"total":2,"distinct":2},"q::rel2":{"total":2,"distinct":2}}}}],"expected_results":[{"answers":{"q":{"type":"choice","choice":"yes","probabilities":{"yes":0.6,"no":0.4},"reliability":{"probes":["reorder","rename"],"option_layout":"sequential","decision":"VERIFY","soft_stability":0.6,"stability":1.0,"rename_stability":1.0,"reorder_stability":1.0,"n_variants":3,"distinct_choices":["yes"],"variant_choices":{"original":"yes","reversed":"yes","letters_reversed":"yes"},"variant_support":{"original":0.6,"reversed":0.6,"letters_reversed":0.6}}}},"usage":{"input_tokens":300,"output_tokens":3,"truncated_questions":["q"],"options":{"q":{"total":2,"distinct":2}}}}]}
],
"calibration_cases": [
{"name":"near-equal-cuts","scores":[0.79996,0.8,0.80004],"correct":[false,false,true],"options":{"targetAccuracy":1.0},"thresholds":[0.80004,0.8],"decisions":["ESCALATE","VERIFY","ACCEPT"]},
{"name":"mixed-neighbors","scores":[0.59996,0.6,0.60004,0.79996,0.8,0.80004],"correct":[false,true,false,true,true,true],"options":{},"thresholds":[0.79996,0.6],"decisions":["ESCALATE","VERIFY","VERIFY","ACCEPT","ACCEPT","ACCEPT"]},
{"name":"equal-score-cohort","scores":[0.8,0.8,0.9],"correct":[true,false,true],"options":{"targetAccuracy":1.0},"thresholds":[0.9,0.0],"decisions":["VERIFY","VERIFY","ACCEPT"]},
{"name":"accept-nothing","scores":[0.2,0.8,0.9],"correct":[false,false,false],"options":{},"thresholds":[1.01,0.9],"decisions":["ESCALATE","ESCALATE","VERIFY"]},
{"name":"all-correct","scores":[0.2,0.8,0.9],"correct":[true,true,true],"options":{},"thresholds":[0.2,0.0],"decisions":["ACCEPT","ACCEPT","ACCEPT"]},
{"name":"edges","scores":[0.0,1.0],"correct":[false,true],"options":{"maxEscalateAccuracy":0.0},"thresholds":[1.0,0.0],"decisions":["VERIFY","ACCEPT"]}
],
"decision_cases": [
{"score":0.0,"accept":null,"escalate":null,"expected":null},
{"score":0.5999999999999999,"accept":null,"escalate":null,"expected":null},
{"score":0.6,"accept":null,"escalate":null,"expected":null},
{"score":0.7999999999999999,"accept":null,"escalate":null,"expected":null},
{"score":0.8,"accept":null,"escalate":null,"expected":null},
{"score":0.8000000000000002,"accept":null,"escalate":null,"expected":null},
{"score":1.0,"accept":null,"escalate":null,"expected":null},
{"score":0.0,"accept":0.8,"escalate":0.6,"expected":"ESCALATE"},
{"score":0.5999999999999999,"accept":0.8,"escalate":0.6,"expected":"ESCALATE"},
{"score":0.6,"accept":0.8,"escalate":0.6,"expected":"VERIFY"},
{"score":0.7999999999999999,"accept":0.8,"escalate":0.6,"expected":"VERIFY"},
{"score":0.8,"accept":0.8,"escalate":0.6,"expected":"ACCEPT"},
{"score":0.8000000000000002,"accept":0.8,"escalate":0.6,"expected":"ACCEPT"},
{"score":1.0,"accept":0.8,"escalate":0.6,"expected":"ACCEPT"},
{"score":0.0,"accept":1.01,"escalate":0.0,"expected":"VERIFY"},
{"score":0.5999999999999999,"accept":1.01,"escalate":0.0,"expected":"VERIFY"},
{"score":0.6,"accept":1.01,"escalate":0.0,"expected":"VERIFY"},
{"score":0.7999999999999999,"accept":1.01,"escalate":0.0,"expected":"VERIFY"},
{"score":0.8,"accept":1.01,"escalate":0.0,"expected":"VERIFY"},
{"score":0.8000000000000002,"accept":1.01,"escalate":0.0,"expected":"VERIFY"},
{"score":1.0,"accept":1.01,"escalate":0.0,"expected":"VERIFY"},
{"score":0.0,"accept":0.0,"escalate":0.0,"expected":"ACCEPT"},
{"score":0.5999999999999999,"accept":0.0,"escalate":0.0,"expected":"ACCEPT"},
{"score":0.6,"accept":0.0,"escalate":0.0,"expected":"ACCEPT"},
{"score":0.7999999999999999,"accept":0.0,"escalate":0.0,"expected":"ACCEPT"},
{"score":0.8,"accept":0.0,"escalate":0.0,"expected":"ACCEPT"},
{"score":0.8000000000000002,"accept":0.0,"escalate":0.0,"expected":"ACCEPT"},
{"score":1.0,"accept":0.0,"escalate":0.0,"expected":"ACCEPT"}
],
"pipeline": {
"cases": [
"below-accept",
"at-accept",
"above-accept"
],
"correct": [
false,
false,
true
],
"options": {
"targetAccuracy": 1.0
},
"thresholds": [
0.8000333333333334,
0.8000000000000002
],
"decisions": [
"ESCALATE",
"VERIFY",
"ACCEPT"
]
}
}