Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(java): advance any round answer, as predict_tournament does
Adeshina Lasisie0f0fb5979f5
diff --git a/.github/workflows/java.yml b/.github/workflows/java.ymlindex 615161a3..98c9a5f6 100644--- a/.github/workflows/java.yml+++ b/.github/workflows/java.yml@@ -154,17 +154,17 @@ jobs:# Without LAYA_CHECKPOINTS the checkpoint-backed factories abort by assumption and the rest# run. `--min-tests` is what stops an aborted suite from looking like a pass.#- # Measured on the merged tree: 1133 register here, 1113 run, 20 abort on a checkpoint this+ # Measured on the merged tree: 1142 register here, 1122 run, 20 abort on a checkpoint this# lane does not have. Exact, not padded -- the floor stood at 680 long enough for the real# figure to pass it by, which is how a floor stops being one. ONE below the JDK-matrix# lane's floor on purpose: that lane passes `-PtestJavaVersion`, which is what makes- # `TestJvmVersionTest` run instead of skip, so it executes 1114. Same suite, one more test,+ # `TestJvmVersionTest` run instead of skip, so it executes 1123. Same suite, one more test,# because the flag enables it. The step below is where the number is justified.- name: Tests that need no modelworking-directory: laya-javarun: ./gradlew --no-daemon test- # Measured on the merged tree, Corretto 17, no checkpoints: 1133 registered, 20 abort,- # 1113 RUN -- the floor counts what ran, so the aborted checkpoint-backed cases are not in+ # Measured on the merged tree, Corretto 17, no checkpoints: 1142 registered, 20 abort,+ # 1122 RUN -- the floor counts what ran, so the aborted checkpoint-backed cases are not in# it. The floor is EXACT, with no margin, which is a change from the margin-of-4 this# replaced. A margin only binds classes larger than itself, and three classes in this lane# run exactly ONE test (`DecoderTest`, `TypedDecisionsParityTest`, `WindowPlanTest`), so no@@ -179,7 +179,7 @@ jobs:if: ${{ !cancelled() }}run: >python laya-java/scripts/check_test_results.py- laya-java/laya-java/build/test-results/test --min-tests 1113 --allow-aborted+ laya-java/laya-java/build/test-results/test --min-tests 1122 --allow-aborted# The samples are single-file source programs, deliberately NOT a Gradle module: a# `samples` subproject would inherit the root build's MavenPublication and@@ -335,7 +335,7 @@ jobs:# dropped, Gradle exits 0 and the floor alone exits 0 on 488 tests, having run JDK 17 twice# and reported it as two JDKs. With `--require-passed` it exits 1.#- # 1114, not the build lane's 1113, and the one is the point: this lane runs one test MORE+ # 1123, not the build lane's 1122, and the one is the point: this lane runs one test MORE# than that one, because `TestJvmVersionTest` is `@EnabledIfSystemProperty` and only# enabled here. Both are EXACT, with no margin, for the reason the build lane gives --# three classes here run exactly one test, so no positive margin can bind them.@@ -356,7 +356,7 @@ jobs:if: ${{ !cancelled() }}run: >python laya-java/scripts/check_test_results.py- laya-java/laya-java/build/test-results/test --min-tests 1114 --allow-aborted+ laya-java/laya-java/build/test-results/test --min-tests 1123 --allow-aborted--require-passed TestJvmVersionTest- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2if: ${{ failure() }}diff --git a/.github/workflows/release-java.yml b/.github/workflows/release-java.ymlindex 774a74b1..f0157d6a 100644--- a/.github/workflows/release-java.yml+++ b/.github/workflows/release-java.yml@@ -67,7 +67,7 @@ jobs:# that lane already covered.- name: Model-free testsrun: ./gradlew --no-daemon test- # 1113 of the 1133 registered tests run without a checkpoint. This floor has been raised+ # 1122 of the 1142 registered tests run without a checkpoint. This floor has been raised# as the model-free suite grew -- 440 left 191 of slack, 625 left 60 -- because a floor# that does not bind is not a gate: under 625 the whole confidence-gate family of 26# tests could have vanished from this lane with the gate still green.@@ -83,7 +83,7 @@ jobs:- name: The suite was not vacuousrun: >python3 scripts/check_test_results.py laya-java/build/test-results/test- --min-tests 1113 --allow-aborted+ --min-tests 1122 --allow-aborted- name: Build the jarsrun: ./gradlew --no-daemon build -x testdiff --git a/laya-java/fixtures/tournament.json b/laya-java/fixtures/tournament.jsonindex c12a0065..a3387841 100644--- a/laya-java/fixtures/tournament.json+++ b/laya-java/fixtures/tournament.json@@ -5505,6 +5505,2952 @@"caller_questions_unmutated": true}],+ "irregular": [+ {+ "name": "dict-out-of-group",+ "group_size": 16,+ "mode": "label",+ "label": "l39",+ "questions": {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "criteria": {+ "l00": "about l00",+ "l01": "about l01",+ "l02": "about l02",+ "l03": "about l03",+ "l04": "about l04",+ "l05": "about l05",+ "l06": "about l06",+ "l07": "about l07",+ "l08": "about l08",+ "l09": "about l09",+ "l10": "about l10",+ "l11": "about l11",+ "l12": "about l12",+ "l13": "about l13",+ "l14": "about l14",+ "l15": "about l15",+ "l16": "about l16",+ "l17": "about l17",+ "l18": "about l18",+ "l19": "about l19",+ "l20": "about l20",+ "l21": "about l21",+ "l22": "about l22",+ "l23": "about l23",+ "l24": "about l24",+ "l25": "about l25",+ "l26": "about l26",+ "l27": "about l27",+ "l28": "about l28",+ "l29": "about l29",+ "l30": "about l30",+ "l31": "about l31",+ "l32": "about l32",+ "l33": "about l33",+ "l34": "about l34",+ "l35": "about l35",+ "l36": "about l36",+ "l37": "about l37",+ "l38": "about l38",+ "l39": "about l39"+ }+ }+ },+ "error": null,+ "answers": {+ "q": "l39"+ },+ "tournament": {+ "q": {+ "labels": [+ "l39",+ "l39",+ "l39"+ ],+ "n": 40,+ "rounds": 1+ }+ },+ "calls": [+ {+ "0": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l00",+ "l01",+ "l02",+ "l03",+ "l04",+ "l05",+ "l06",+ "l07",+ "l08",+ "l09",+ "l10",+ "l11",+ "l12"+ ],+ "criteria": {+ "l00": "about l00",+ "l01": "about l01",+ "l02": "about l02",+ "l03": "about l03",+ "l04": "about l04",+ "l05": "about l05",+ "l06": "about l06",+ "l07": "about l07",+ "l08": "about l08",+ "l09": "about l09",+ "l10": "about l10",+ "l11": "about l11",+ "l12": "about l12"+ }+ },+ "1": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l13",+ "l14",+ "l15",+ "l16",+ "l17",+ "l18",+ "l19",+ "l20",+ "l21",+ "l22",+ "l23",+ "l24",+ "l25"+ ],+ "criteria": {+ "l13": "about l13",+ "l14": "about l14",+ "l15": "about l15",+ "l16": "about l16",+ "l17": "about l17",+ "l18": "about l18",+ "l19": "about l19",+ "l20": "about l20",+ "l21": "about l21",+ "l22": "about l22",+ "l23": "about l23",+ "l24": "about l24",+ "l25": "about l25"+ }+ },+ "2": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l26",+ "l27",+ "l28",+ "l29",+ "l30",+ "l31",+ "l32",+ "l33",+ "l34",+ "l35",+ "l36",+ "l37",+ "l38",+ "l39"+ ],+ "criteria": {+ "l26": "about l26",+ "l27": "about l27",+ "l28": "about l28",+ "l29": "about l29",+ "l30": "about l30",+ "l31": "about l31",+ "l32": "about l32",+ "l33": "about l33",+ "l34": "about l34",+ "l35": "about l35",+ "l36": "about l36",+ "l37": "about l37",+ "l38": "about l38",+ "l39": "about l39"+ }+ }+ },+ {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l39"+ ],+ "criteria": {+ "l39": "about l39"+ }+ }+ }+ ]+ },+ {+ "name": "list-out-of-group",+ "group_size": 16,+ "mode": "label",+ "label": "l39",+ "questions": {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "criteria": [+ "l00",+ "l01",+ "l02",+ "l03",+ "l04",+ "l05",+ "l06",+ "l07",+ "l08",+ "l09",+ "l10",+ "l11",+ "l12",+ "l13",+ "l14",+ "l15",+ "l16",+ "l17",+ "l18",+ "l19",+ "l20",+ "l21",+ "l22",+ "l23",+ "l24",+ "l25",+ "l26",+ "l27",+ "l28",+ "l29",+ "l30",+ "l31",+ "l32",+ "l33",+ "l34",+ "l35",+ "l36",+ "l37",+ "l38",+ "l39"+ ]+ }+ },+ "error": null,+ "answers": {+ "q": "l39"+ },+ "tournament": {+ "q": {+ "labels": [+ "l39",+ "l39",+ "l39"+ ],+ "n": 40,+ "rounds": 1+ }+ },+ "calls": [+ {+ "0": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l00",+ "l01",+ "l02",+ "l03",+ "l04",+ "l05",+ "l06",+ "l07",+ "l08",+ "l09",+ "l10",+ "l11",+ "l12"+ ]+ },+ "1": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l13",+ "l14",+ "l15",+ "l16",+ "l17",+ "l18",+ "l19",+ "l20",+ "l21",+ "l22",+ "l23",+ "l24",+ "l25"+ ]+ },+ "2": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l26",+ "l27",+ "l28",+ "l29",+ "l30",+ "l31",+ "l32",+ "l33",+ "l34",+ "l35",+ "l36",+ "l37",+ "l38",+ "l39"+ ]+ }+ },+ {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l39",+ "l39",+ "l39"+ ]+ }+ }+ ]+ },+ {+ "name": "dict-non-label",+ "group_size": 16,+ "mode": "label",+ "label": "zzz",+ "questions": {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "criteria": {+ "l00": "about l00",+ "l01": "about l01",+ "l02": "about l02",+ "l03": "about l03",+ "l04": "about l04",+ "l05": "about l05",+ "l06": "about l06",+ "l07": "about l07",+ "l08": "about l08",+ "l09": "about l09",+ "l10": "about l10",+ "l11": "about l11",+ "l12": "about l12",+ "l13": "about l13",+ "l14": "about l14",+ "l15": "about l15",+ "l16": "about l16",+ "l17": "about l17",+ "l18": "about l18",+ "l19": "about l19",+ "l20": "about l20",+ "l21": "about l21",+ "l22": "about l22",+ "l23": "about l23",+ "l24": "about l24",+ "l25": "about l25",+ "l26": "about l26",+ "l27": "about l27",+ "l28": "about l28",+ "l29": "about l29",+ "l30": "about l30",+ "l31": "about l31",+ "l32": "about l32",+ "l33": "about l33",+ "l34": "about l34",+ "l35": "about l35",+ "l36": "about l36",+ "l37": "about l37",+ "l38": "about l38",+ "l39": "about l39"+ }+ }+ },+ "error": "KeyError",+ "message": "'zzz'",+ "calls": [+ {+ "0": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l00",+ "l01",+ "l02",+ "l03",+ "l04",+ "l05",+ "l06",+ "l07",+ "l08",+ "l09",+ "l10",+ "l11",+ "l12"+ ],+ "criteria": {+ "l00": "about l00",+ "l01": "about l01",+ "l02": "about l02",+ "l03": "about l03",+ "l04": "about l04",+ "l05": "about l05",+ "l06": "about l06",+ "l07": "about l07",+ "l08": "about l08",+ "l09": "about l09",+ "l10": "about l10",+ "l11": "about l11",+ "l12": "about l12"+ }+ },+ "1": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l13",+ "l14",+ "l15",+ "l16",+ "l17",+ "l18",+ "l19",+ "l20",+ "l21",+ "l22",+ "l23",+ "l24",+ "l25"+ ],+ "criteria": {+ "l13": "about l13",+ "l14": "about l14",+ "l15": "about l15",+ "l16": "about l16",+ "l17": "about l17",+ "l18": "about l18",+ "l19": "about l19",+ "l20": "about l20",+ "l21": "about l21",+ "l22": "about l22",+ "l23": "about l23",+ "l24": "about l24",+ "l25": "about l25"+ }+ },+ "2": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l26",+ "l27",+ "l28",+ "l29",+ "l30",+ "l31",+ "l32",+ "l33",+ "l34",+ "l35",+ "l36",+ "l37",+ "l38",+ "l39"+ ],+ "criteria": {+ "l26": "about l26",+ "l27": "about l27",+ "l28": "about l28",+ "l29": "about l29",+ "l30": "about l30",+ "l31": "about l31",+ "l32": "about l32",+ "l33": "about l33",+ "l34": "about l34",+ "l35": "about l35",+ "l36": "about l36",+ "l37": "about l37",+ "l38": "about l38",+ "l39": "about l39"+ }+ }+ }+ ]+ },+ {+ "name": "list-non-label",+ "group_size": 16,+ "mode": "label",+ "label": "zzz",+ "questions": {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "criteria": [+ "l00",+ "l01",+ "l02",+ "l03",+ "l04",+ "l05",+ "l06",+ "l07",+ "l08",+ "l09",+ "l10",+ "l11",+ "l12",+ "l13",+ "l14",+ "l15",+ "l16",+ "l17",+ "l18",+ "l19",+ "l20",+ "l21",+ "l22",+ "l23",+ "l24",+ "l25",+ "l26",+ "l27",+ "l28",+ "l29",+ "l30",+ "l31",+ "l32",+ "l33",+ "l34",+ "l35",+ "l36",+ "l37",+ "l38",+ "l39"+ ]+ }+ },+ "error": null,+ "answers": {+ "q": "zzz"+ },+ "tournament": {+ "q": {+ "labels": [+ "zzz",+ "zzz",+ "zzz"+ ],+ "n": 40,+ "rounds": 1+ }+ },+ "calls": [+ {+ "0": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l00",+ "l01",+ "l02",+ "l03",+ "l04",+ "l05",+ "l06",+ "l07",+ "l08",+ "l09",+ "l10",+ "l11",+ "l12"+ ]+ },+ "1": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l13",+ "l14",+ "l15",+ "l16",+ "l17",+ "l18",+ "l19",+ "l20",+ "l21",+ "l22",+ "l23",+ "l24",+ "l25"+ ]+ },+ "2": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l26",+ "l27",+ "l28",+ "l29",+ "l30",+ "l31",+ "l32",+ "l33",+ "l34",+ "l35",+ "l36",+ "l37",+ "l38",+ "l39"+ ]+ }+ },+ {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "zzz",+ "zzz",+ "zzz"+ ]+ }+ }+ ]+ },+ {+ "name": "dict-non-label-next-round",+ "group_size": 16,+ "mode": "label",+ "label": "zzz",+ "questions": {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "criteria": {+ "l000": "about l000",+ "l001": "about l001",+ "l002": "about l002",+ "l003": "about l003",+ "l004": "about l004",+ "l005": "about l005",+ "l006": "about l006",+ "l007": "about l007",+ "l008": "about l008",+ "l009": "about l009",+ "l010": "about l010",+ "l011": "about l011",+ "l012": "about l012",+ "l013": "about l013",+ "l014": "about l014",+ "l015": "about l015",+ "l016": "about l016",+ "l017": "about l017",+ "l018": "about l018",+ "l019": "about l019",+ "l020": "about l020",+ "l021": "about l021",+ "l022": "about l022",+ "l023": "about l023",+ "l024": "about l024",+ "l025": "about l025",+ "l026": "about l026",+ "l027": "about l027",+ "l028": "about l028",+ "l029": "about l029",+ "l030": "about l030",+ "l031": "about l031",+ "l032": "about l032",+ "l033": "about l033",+ "l034": "about l034",+ "l035": "about l035",+ "l036": "about l036",+ "l037": "about l037",+ "l038": "about l038",+ "l039": "about l039",+ "l040": "about l040",+ "l041": "about l041",+ "l042": "about l042",+ "l043": "about l043",+ "l044": "about l044",+ "l045": "about l045",+ "l046": "about l046",+ "l047": "about l047",+ "l048": "about l048",+ "l049": "about l049",+ "l050": "about l050",+ "l051": "about l051",+ "l052": "about l052",+ "l053": "about l053",+ "l054": "about l054",+ "l055": "about l055",+ "l056": "about l056",+ "l057": "about l057",+ "l058": "about l058",+ "l059": "about l059",+ "l060": "about l060",+ "l061": "about l061",+ "l062": "about l062",+ "l063": "about l063",+ "l064": "about l064",+ "l065": "about l065",+ "l066": "about l066",+ "l067": "about l067",+ "l068": "about l068",+ "l069": "about l069",+ "l070": "about l070",+ "l071": "about l071",+ "l072": "about l072",+ "l073": "about l073",+ "l074": "about l074",+ "l075": "about l075",+ "l076": "about l076",+ "l077": "about l077",+ "l078": "about l078",+ "l079": "about l079",+ "l080": "about l080",+ "l081": "about l081",+ "l082": "about l082",+ "l083": "about l083",+ "l084": "about l084",+ "l085": "about l085",+ "l086": "about l086",+ "l087": "about l087",+ "l088": "about l088",+ "l089": "about l089",+ "l090": "about l090",+ "l091": "about l091",+ "l092": "about l092",+ "l093": "about l093",+ "l094": "about l094",+ "l095": "about l095",+ "l096": "about l096",+ "l097": "about l097",+ "l098": "about l098",+ "l099": "about l099",+ "l100": "about l100",+ "l101": "about l101",+ "l102": "about l102",+ "l103": "about l103",+ "l104": "about l104",+ "l105": "about l105",+ "l106": "about l106",+ "l107": "about l107",+ "l108": "about l108",+ "l109": "about l109",+ "l110": "about l110",+ "l111": "about l111",+ "l112": "about l112",+ "l113": "about l113",+ "l114": "about l114",+ "l115": "about l115",+ "l116": "about l116",+ "l117": "about l117",+ "l118": "about l118",+ "l119": "about l119",+ "l120": "about l120",+ "l121": "about l121",+ "l122": "about l122",+ "l123": "about l123",+ "l124": "about l124",+ "l125": "about l125",+ "l126": "about l126",+ "l127": "about l127",+ "l128": "about l128",+ "l129": "about l129",+ "l130": "about l130",+ "l131": "about l131",+ "l132": "about l132",+ "l133": "about l133",+ "l134": "about l134",+ "l135": "about l135",+ "l136": "about l136",+ "l137": "about l137",+ "l138": "about l138",+ "l139": "about l139",+ "l140": "about l140",+ "l141": "about l141",+ "l142": "about l142",+ "l143": "about l143",+ "l144": "about l144",+ "l145": "about l145",+ "l146": "about l146",+ "l147": "about l147",+ "l148": "about l148",+ "l149": "about l149",+ "l150": "about l150",+ "l151": "about l151",+ "l152": "about l152",+ "l153": "about l153",+ "l154": "about l154",+ "l155": "about l155",+ "l156": "about l156",+ "l157": "about l157",+ "l158": "about l158",+ "l159": "about l159",+ "l160": "about l160",+ "l161": "about l161",+ "l162": "about l162",+ "l163": "about l163",+ "l164": "about l164",+ "l165": "about l165",+ "l166": "about l166",+ "l167": "about l167",+ "l168": "about l168",+ "l169": "about l169",+ "l170": "about l170",+ "l171": "about l171",+ "l172": "about l172",+ "l173": "about l173",+ "l174": "about l174",+ "l175": "about l175",+ "l176": "about l176",+ "l177": "about l177",+ "l178": "about l178",+ "l179": "about l179",+ "l180": "about l180",+ "l181": "about l181",+ "l182": "about l182",+ "l183": "about l183",+ "l184": "about l184",+ "l185": "about l185",+ "l186": "about l186",+ "l187": "about l187",+ "l188": "about l188",+ "l189": "about l189",+ "l190": "about l190",+ "l191": "about l191",+ "l192": "about l192",+ "l193": "about l193",+ "l194": "about l194",+ "l195": "about l195",+ "l196": "about l196",+ "l197": "about l197",+ "l198": "about l198",+ "l199": "about l199",+ "l200": "about l200",+ "l201": "about l201",+ "l202": "about l202",+ "l203": "about l203",+ "l204": "about l204",+ "l205": "about l205",+ "l206": "about l206",+ "l207": "about l207",+ "l208": "about l208",+ "l209": "about l209",+ "l210": "about l210",+ "l211": "about l211",+ "l212": "about l212",+ "l213": "about l213",+ "l214": "about l214",+ "l215": "about l215",+ "l216": "about l216",+ "l217": "about l217",+ "l218": "about l218",+ "l219": "about l219",+ "l220": "about l220",+ "l221": "about l221",+ "l222": "about l222",+ "l223": "about l223",+ "l224": "about l224",+ "l225": "about l225",+ "l226": "about l226",+ "l227": "about l227",+ "l228": "about l228",+ "l229": "about l229",+ "l230": "about l230",+ "l231": "about l231",+ "l232": "about l232",+ "l233": "about l233",+ "l234": "about l234",+ "l235": "about l235",+ "l236": "about l236",+ "l237": "about l237",+ "l238": "about l238",+ "l239": "about l239",+ "l240": "about l240",+ "l241": "about l241",+ "l242": "about l242",+ "l243": "about l243",+ "l244": "about l244",+ "l245": "about l245",+ "l246": "about l246",+ "l247": "about l247",+ "l248": "about l248",+ "l249": "about l249",+ "l250": "about l250",+ "l251": "about l251",+ "l252": "about l252",+ "l253": "about l253",+ "l254": "about l254",+ "l255": "about l255",+ "l256": "about l256",+ "l257": "about l257",+ "l258": "about l258",+ "l259": "about l259",+ "l260": "about l260",+ "l261": "about l261",+ "l262": "about l262",+ "l263": "about l263",+ "l264": "about l264",+ "l265": "about l265",+ "l266": "about l266",+ "l267": "about l267",+ "l268": "about l268",+ "l269": "about l269",+ "l270": "about l270",+ "l271": "about l271",+ "l272": "about l272",+ "l273": "about l273",+ "l274": "about l274",+ "l275": "about l275",+ "l276": "about l276",+ "l277": "about l277",+ "l278": "about l278",+ "l279": "about l279",+ "l280": "about l280",+ "l281": "about l281",+ "l282": "about l282",+ "l283": "about l283",+ "l284": "about l284",+ "l285": "about l285",+ "l286": "about l286",+ "l287": "about l287",+ "l288": "about l288",+ "l289": "about l289",+ "l290": "about l290",+ "l291": "about l291",+ "l292": "about l292",+ "l293": "about l293",+ "l294": "about l294",+ "l295": "about l295",+ "l296": "about l296",+ "l297": "about l297",+ "l298": "about l298",+ "l299": "about l299"+ }+ }+ },+ "error": "KeyError",+ "message": "'zzz'",+ "calls": [+ {+ "0": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l000",+ "l001",+ "l002",+ "l003",+ "l004",+ "l005",+ "l006",+ "l007",+ "l008",+ "l009",+ "l010",+ "l011",+ "l012",+ "l013",+ "l014"+ ],+ "criteria": {+ "l000": "about l000",+ "l001": "about l001",+ "l002": "about l002",+ "l003": "about l003",+ "l004": "about l004",+ "l005": "about l005",+ "l006": "about l006",+ "l007": "about l007",+ "l008": "about l008",+ "l009": "about l009",+ "l010": "about l010",+ "l011": "about l011",+ "l012": "about l012",+ "l013": "about l013",+ "l014": "about l014"+ }+ },+ "1": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l015",+ "l016",+ "l017",+ "l018",+ "l019",+ "l020",+ "l021",+ "l022",+ "l023",+ "l024",+ "l025",+ "l026",+ "l027",+ "l028",+ "l029",+ "l030"+ ],+ "criteria": {+ "l015": "about l015",+ "l016": "about l016",+ "l017": "about l017",+ "l018": "about l018",+ "l019": "about l019",+ "l020": "about l020",+ "l021": "about l021",+ "l022": "about l022",+ "l023": "about l023",+ "l024": "about l024",+ "l025": "about l025",+ "l026": "about l026",+ "l027": "about l027",+ "l028": "about l028",+ "l029": "about l029",+ "l030": "about l030"+ }+ },+ "2": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l031",+ "l032",+ "l033",+ "l034",+ "l035",+ "l036",+ "l037",+ "l038",+ "l039",+ "l040",+ "l041",+ "l042",+ "l043",+ "l044",+ "l045",+ "l046"+ ],+ "criteria": {+ "l031": "about l031",+ "l032": "about l032",+ "l033": "about l033",+ "l034": "about l034",+ "l035": "about l035",+ "l036": "about l036",+ "l037": "about l037",+ "l038": "about l038",+ "l039": "about l039",+ "l040": "about l040",+ "l041": "about l041",+ "l042": "about l042",+ "l043": "about l043",+ "l044": "about l044",+ "l045": "about l045",+ "l046": "about l046"+ }+ },+ "3": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l047",+ "l048",+ "l049",+ "l050",+ "l051",+ "l052",+ "l053",+ "l054",+ "l055",+ "l056",+ "l057",+ "l058",+ "l059",+ "l060",+ "l061",+ "l062"+ ],+ "criteria": {+ "l047": "about l047",+ "l048": "about l048",+ "l049": "about l049",+ "l050": "about l050",+ "l051": "about l051",+ "l052": "about l052",+ "l053": "about l053",+ "l054": "about l054",+ "l055": "about l055",+ "l056": "about l056",+ "l057": "about l057",+ "l058": "about l058",+ "l059": "about l059",+ "l060": "about l060",+ "l061": "about l061",+ "l062": "about l062"+ }+ },+ "4": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l063",+ "l064",+ "l065",+ "l066",+ "l067",+ "l068",+ "l069",+ "l070",+ "l071",+ "l072",+ "l073",+ "l074",+ "l075",+ "l076",+ "l077"+ ],+ "criteria": {+ "l063": "about l063",+ "l064": "about l064",+ "l065": "about l065",+ "l066": "about l066",+ "l067": "about l067",+ "l068": "about l068",+ "l069": "about l069",+ "l070": "about l070",+ "l071": "about l071",+ "l072": "about l072",+ "l073": "about l073",+ "l074": "about l074",+ "l075": "about l075",+ "l076": "about l076",+ "l077": "about l077"+ }+ },+ "5": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l078",+ "l079",+ "l080",+ "l081",+ "l082",+ "l083",+ "l084",+ "l085",+ "l086",+ "l087",+ "l088",+ "l089",+ "l090",+ "l091",+ "l092",+ "l093"+ ],+ "criteria": {+ "l078": "about l078",+ "l079": "about l079",+ "l080": "about l080",+ "l081": "about l081",+ "l082": "about l082",+ "l083": "about l083",+ "l084": "about l084",+ "l085": "about l085",+ "l086": "about l086",+ "l087": "about l087",+ "l088": "about l088",+ "l089": "about l089",+ "l090": "about l090",+ "l091": "about l091",+ "l092": "about l092",+ "l093": "about l093"+ }+ },+ "6": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l094",+ "l095",+ "l096",+ "l097",+ "l098",+ "l099",+ "l100",+ "l101",+ "l102",+ "l103",+ "l104",+ "l105",+ "l106",+ "l107",+ "l108",+ "l109"+ ],+ "criteria": {+ "l094": "about l094",+ "l095": "about l095",+ "l096": "about l096",+ "l097": "about l097",+ "l098": "about l098",+ "l099": "about l099",+ "l100": "about l100",+ "l101": "about l101",+ "l102": "about l102",+ "l103": "about l103",+ "l104": "about l104",+ "l105": "about l105",+ "l106": "about l106",+ "l107": "about l107",+ "l108": "about l108",+ "l109": "about l109"+ }+ },+ "7": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l110",+ "l111",+ "l112",+ "l113",+ "l114",+ "l115",+ "l116",+ "l117",+ "l118",+ "l119",+ "l120",+ "l121",+ "l122",+ "l123",+ "l124",+ "l125"+ ],+ "criteria": {+ "l110": "about l110",+ "l111": "about l111",+ "l112": "about l112",+ "l113": "about l113",+ "l114": "about l114",+ "l115": "about l115",+ "l116": "about l116",+ "l117": "about l117",+ "l118": "about l118",+ "l119": "about l119",+ "l120": "about l120",+ "l121": "about l121",+ "l122": "about l122",+ "l123": "about l123",+ "l124": "about l124",+ "l125": "about l125"+ }+ },+ "8": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l126",+ "l127",+ "l128",+ "l129",+ "l130",+ "l131",+ "l132",+ "l133",+ "l134",+ "l135",+ "l136",+ "l137",+ "l138",+ "l139",+ "l140",+ "l141"+ ],+ "criteria": {+ "l126": "about l126",+ "l127": "about l127",+ "l128": "about l128",+ "l129": "about l129",+ "l130": "about l130",+ "l131": "about l131",+ "l132": "about l132",+ "l133": "about l133",+ "l134": "about l134",+ "l135": "about l135",+ "l136": "about l136",+ "l137": "about l137",+ "l138": "about l138",+ "l139": "about l139",+ "l140": "about l140",+ "l141": "about l141"+ }+ },+ "9": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l142",+ "l143",+ "l144",+ "l145",+ "l146",+ "l147",+ "l148",+ "l149",+ "l150",+ "l151",+ "l152",+ "l153",+ "l154",+ "l155",+ "l156"+ ],+ "criteria": {+ "l142": "about l142",+ "l143": "about l143",+ "l144": "about l144",+ "l145": "about l145",+ "l146": "about l146",+ "l147": "about l147",+ "l148": "about l148",+ "l149": "about l149",+ "l150": "about l150",+ "l151": "about l151",+ "l152": "about l152",+ "l153": "about l153",+ "l154": "about l154",+ "l155": "about l155",+ "l156": "about l156"+ }+ },+ "10": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l157",+ "l158",+ "l159",+ "l160",+ "l161",+ "l162",+ "l163",+ "l164",+ "l165",+ "l166",+ "l167",+ "l168",+ "l169",+ "l170",+ "l171",+ "l172"+ ],+ "criteria": {+ "l157": "about l157",+ "l158": "about l158",+ "l159": "about l159",+ "l160": "about l160",+ "l161": "about l161",+ "l162": "about l162",+ "l163": "about l163",+ "l164": "about l164",+ "l165": "about l165",+ "l166": "about l166",+ "l167": "about l167",+ "l168": "about l168",+ "l169": "about l169",+ "l170": "about l170",+ "l171": "about l171",+ "l172": "about l172"+ }+ },+ "11": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l173",+ "l174",+ "l175",+ "l176",+ "l177",+ "l178",+ "l179",+ "l180",+ "l181",+ "l182",+ "l183",+ "l184",+ "l185",+ "l186",+ "l187",+ "l188"+ ],+ "criteria": {+ "l173": "about l173",+ "l174": "about l174",+ "l175": "about l175",+ "l176": "about l176",+ "l177": "about l177",+ "l178": "about l178",+ "l179": "about l179",+ "l180": "about l180",+ "l181": "about l181",+ "l182": "about l182",+ "l183": "about l183",+ "l184": "about l184",+ "l185": "about l185",+ "l186": "about l186",+ "l187": "about l187",+ "l188": "about l188"+ }+ },+ "12": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l189",+ "l190",+ "l191",+ "l192",+ "l193",+ "l194",+ "l195",+ "l196",+ "l197",+ "l198",+ "l199",+ "l200",+ "l201",+ "l202",+ "l203",+ "l204"+ ],+ "criteria": {+ "l189": "about l189",+ "l190": "about l190",+ "l191": "about l191",+ "l192": "about l192",+ "l193": "about l193",+ "l194": "about l194",+ "l195": "about l195",+ "l196": "about l196",+ "l197": "about l197",+ "l198": "about l198",+ "l199": "about l199",+ "l200": "about l200",+ "l201": "about l201",+ "l202": "about l202",+ "l203": "about l203",+ "l204": "about l204"+ }+ },+ "13": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l205",+ "l206",+ "l207",+ "l208",+ "l209",+ "l210",+ "l211",+ "l212",+ "l213",+ "l214",+ "l215",+ "l216",+ "l217",+ "l218",+ "l219",+ "l220"+ ],+ "criteria": {+ "l205": "about l205",+ "l206": "about l206",+ "l207": "about l207",+ "l208": "about l208",+ "l209": "about l209",+ "l210": "about l210",+ "l211": "about l211",+ "l212": "about l212",+ "l213": "about l213",+ "l214": "about l214",+ "l215": "about l215",+ "l216": "about l216",+ "l217": "about l217",+ "l218": "about l218",+ "l219": "about l219",+ "l220": "about l220"+ }+ },+ "14": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l221",+ "l222",+ "l223",+ "l224",+ "l225",+ "l226",+ "l227",+ "l228",+ "l229",+ "l230",+ "l231",+ "l232",+ "l233",+ "l234",+ "l235"+ ],+ "criteria": {+ "l221": "about l221",+ "l222": "about l222",+ "l223": "about l223",+ "l224": "about l224",+ "l225": "about l225",+ "l226": "about l226",+ "l227": "about l227",+ "l228": "about l228",+ "l229": "about l229",+ "l230": "about l230",+ "l231": "about l231",+ "l232": "about l232",+ "l233": "about l233",+ "l234": "about l234",+ "l235": "about l235"+ }+ },+ "15": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l236",+ "l237",+ "l238",+ "l239",+ "l240",+ "l241",+ "l242",+ "l243",+ "l244",+ "l245",+ "l246",+ "l247",+ "l248",+ "l249",+ "l250",+ "l251"+ ],+ "criteria": {+ "l236": "about l236",+ "l237": "about l237",+ "l238": "about l238",+ "l239": "about l239",+ "l240": "about l240",+ "l241": "about l241",+ "l242": "about l242",+ "l243": "about l243",+ "l244": "about l244",+ "l245": "about l245",+ "l246": "about l246",+ "l247": "about l247",+ "l248": "about l248",+ "l249": "about l249",+ "l250": "about l250",+ "l251": "about l251"+ }+ },+ "16": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l252",+ "l253",+ "l254",+ "l255",+ "l256",+ "l257",+ "l258",+ "l259",+ "l260",+ "l261",+ "l262",+ "l263",+ "l264",+ "l265",+ "l266",+ "l267"+ ],+ "criteria": {+ "l252": "about l252",+ "l253": "about l253",+ "l254": "about l254",+ "l255": "about l255",+ "l256": "about l256",+ "l257": "about l257",+ "l258": "about l258",+ "l259": "about l259",+ "l260": "about l260",+ "l261": "about l261",+ "l262": "about l262",+ "l263": "about l263",+ "l264": "about l264",+ "l265": "about l265",+ "l266": "about l266",+ "l267": "about l267"+ }+ },+ "17": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l268",+ "l269",+ "l270",+ "l271",+ "l272",+ "l273",+ "l274",+ "l275",+ "l276",+ "l277",+ "l278",+ "l279",+ "l280",+ "l281",+ "l282",+ "l283"+ ],+ "criteria": {+ "l268": "about l268",+ "l269": "about l269",+ "l270": "about l270",+ "l271": "about l271",+ "l272": "about l272",+ "l273": "about l273",+ "l274": "about l274",+ "l275": "about l275",+ "l276": "about l276",+ "l277": "about l277",+ "l278": "about l278",+ "l279": "about l279",+ "l280": "about l280",+ "l281": "about l281",+ "l282": "about l282",+ "l283": "about l283"+ }+ },+ "18": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l284",+ "l285",+ "l286",+ "l287",+ "l288",+ "l289",+ "l290",+ "l291",+ "l292",+ "l293",+ "l294",+ "l295",+ "l296",+ "l297",+ "l298",+ "l299"+ ],+ "criteria": {+ "l284": "about l284",+ "l285": "about l285",+ "l286": "about l286",+ "l287": "about l287",+ "l288": "about l288",+ "l289": "about l289",+ "l290": "about l290",+ "l291": "about l291",+ "l292": "about l292",+ "l293": "about l293",+ "l294": "about l294",+ "l295": "about l295",+ "l296": "about l296",+ "l297": "about l297",+ "l298": "about l298",+ "l299": "about l299"+ }+ }+ }+ ]+ },+ {+ "name": "list-non-label-two-rounds",+ "group_size": 16,+ "mode": "label",+ "label": "zzz",+ "questions": {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "criteria": [+ "l000",+ "l001",+ "l002",+ "l003",+ "l004",+ "l005",+ "l006",+ "l007",+ "l008",+ "l009",+ "l010",+ "l011",+ "l012",+ "l013",+ "l014",+ "l015",+ "l016",+ "l017",+ "l018",+ "l019",+ "l020",+ "l021",+ "l022",+ "l023",+ "l024",+ "l025",+ "l026",+ "l027",+ "l028",+ "l029",+ "l030",+ "l031",+ "l032",+ "l033",+ "l034",+ "l035",+ "l036",+ "l037",+ "l038",+ "l039",+ "l040",+ "l041",+ "l042",+ "l043",+ "l044",+ "l045",+ "l046",+ "l047",+ "l048",+ "l049",+ "l050",+ "l051",+ "l052",+ "l053",+ "l054",+ "l055",+ "l056",+ "l057",+ "l058",+ "l059",+ "l060",+ "l061",+ "l062",+ "l063",+ "l064",+ "l065",+ "l066",+ "l067",+ "l068",+ "l069",+ "l070",+ "l071",+ "l072",+ "l073",+ "l074",+ "l075",+ "l076",+ "l077",+ "l078",+ "l079",+ "l080",+ "l081",+ "l082",+ "l083",+ "l084",+ "l085",+ "l086",+ "l087",+ "l088",+ "l089",+ "l090",+ "l091",+ "l092",+ "l093",+ "l094",+ "l095",+ "l096",+ "l097",+ "l098",+ "l099",+ "l100",+ "l101",+ "l102",+ "l103",+ "l104",+ "l105",+ "l106",+ "l107",+ "l108",+ "l109",+ "l110",+ "l111",+ "l112",+ "l113",+ "l114",+ "l115",+ "l116",+ "l117",+ "l118",+ "l119",+ "l120",+ "l121",+ "l122",+ "l123",+ "l124",+ "l125",+ "l126",+ "l127",+ "l128",+ "l129",+ "l130",+ "l131",+ "l132",+ "l133",+ "l134",+ "l135",+ "l136",+ "l137",+ "l138",+ "l139",+ "l140",+ "l141",+ "l142",+ "l143",+ "l144",+ "l145",+ "l146",+ "l147",+ "l148",+ "l149",+ "l150",+ "l151",+ "l152",+ "l153",+ "l154",+ "l155",+ "l156",+ "l157",+ "l158",+ "l159",+ "l160",+ "l161",+ "l162",+ "l163",+ "l164",+ "l165",+ "l166",+ "l167",+ "l168",+ "l169",+ "l170",+ "l171",+ "l172",+ "l173",+ "l174",+ "l175",+ "l176",+ "l177",+ "l178",+ "l179",+ "l180",+ "l181",+ "l182",+ "l183",+ "l184",+ "l185",+ "l186",+ "l187",+ "l188",+ "l189",+ "l190",+ "l191",+ "l192",+ "l193",+ "l194",+ "l195",+ "l196",+ "l197",+ "l198",+ "l199",+ "l200",+ "l201",+ "l202",+ "l203",+ "l204",+ "l205",+ "l206",+ "l207",+ "l208",+ "l209",+ "l210",+ "l211",+ "l212",+ "l213",+ "l214",+ "l215",+ "l216",+ "l217",+ "l218",+ "l219",+ "l220",+ "l221",+ "l222",+ "l223",+ "l224",+ "l225",+ "l226",+ "l227",+ "l228",+ "l229",+ "l230",+ "l231",+ "l232",+ "l233",+ "l234",+ "l235",+ "l236",+ "l237",+ "l238",+ "l239",+ "l240",+ "l241",+ "l242",+ "l243",+ "l244",+ "l245",+ "l246",+ "l247",+ "l248",+ "l249",+ "l250",+ "l251",+ "l252",+ "l253",+ "l254",+ "l255",+ "l256",+ "l257",+ "l258",+ "l259",+ "l260",+ "l261",+ "l262",+ "l263",+ "l264",+ "l265",+ "l266",+ "l267",+ "l268",+ "l269",+ "l270",+ "l271",+ "l272",+ "l273",+ "l274",+ "l275",+ "l276",+ "l277",+ "l278",+ "l279",+ "l280",+ "l281",+ "l282",+ "l283",+ "l284",+ "l285",+ "l286",+ "l287",+ "l288",+ "l289",+ "l290",+ "l291",+ "l292",+ "l293",+ "l294",+ "l295",+ "l296",+ "l297",+ "l298",+ "l299"+ ]+ }+ },+ "error": null,+ "answers": {+ "q": "zzz"+ },+ "tournament": {+ "q": {+ "labels": [+ "zzz",+ "zzz"+ ],+ "n": 300,+ "rounds": 2+ }+ },+ "calls": [+ {+ "0": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l000",+ "l001",+ "l002",+ "l003",+ "l004",+ "l005",+ "l006",+ "l007",+ "l008",+ "l009",+ "l010",+ "l011",+ "l012",+ "l013",+ "l014"+ ]+ },+ "1": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l015",+ "l016",+ "l017",+ "l018",+ "l019",+ "l020",+ "l021",+ "l022",+ "l023",+ "l024",+ "l025",+ "l026",+ "l027",+ "l028",+ "l029",+ "l030"+ ]+ },+ "2": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l031",+ "l032",+ "l033",+ "l034",+ "l035",+ "l036",+ "l037",+ "l038",+ "l039",+ "l040",+ "l041",+ "l042",+ "l043",+ "l044",+ "l045",+ "l046"+ ]+ },+ "3": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l047",+ "l048",+ "l049",+ "l050",+ "l051",+ "l052",+ "l053",+ "l054",+ "l055",+ "l056",+ "l057",+ "l058",+ "l059",+ "l060",+ "l061",+ "l062"+ ]+ },+ "4": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l063",+ "l064",+ "l065",+ "l066",+ "l067",+ "l068",+ "l069",+ "l070",+ "l071",+ "l072",+ "l073",+ "l074",+ "l075",+ "l076",+ "l077"+ ]+ },+ "5": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l078",+ "l079",+ "l080",+ "l081",+ "l082",+ "l083",+ "l084",+ "l085",+ "l086",+ "l087",+ "l088",+ "l089",+ "l090",+ "l091",+ "l092",+ "l093"+ ]+ },+ "6": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l094",+ "l095",+ "l096",+ "l097",+ "l098",+ "l099",+ "l100",+ "l101",+ "l102",+ "l103",+ "l104",+ "l105",+ "l106",+ "l107",+ "l108",+ "l109"+ ]+ },+ "7": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l110",+ "l111",+ "l112",+ "l113",+ "l114",+ "l115",+ "l116",+ "l117",+ "l118",+ "l119",+ "l120",+ "l121",+ "l122",+ "l123",+ "l124",+ "l125"+ ]+ },+ "8": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l126",+ "l127",+ "l128",+ "l129",+ "l130",+ "l131",+ "l132",+ "l133",+ "l134",+ "l135",+ "l136",+ "l137",+ "l138",+ "l139",+ "l140",+ "l141"+ ]+ },+ "9": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l142",+ "l143",+ "l144",+ "l145",+ "l146",+ "l147",+ "l148",+ "l149",+ "l150",+ "l151",+ "l152",+ "l153",+ "l154",+ "l155",+ "l156"+ ]+ },+ "10": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l157",+ "l158",+ "l159",+ "l160",+ "l161",+ "l162",+ "l163",+ "l164",+ "l165",+ "l166",+ "l167",+ "l168",+ "l169",+ "l170",+ "l171",+ "l172"+ ]+ },+ "11": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l173",+ "l174",+ "l175",+ "l176",+ "l177",+ "l178",+ "l179",+ "l180",+ "l181",+ "l182",+ "l183",+ "l184",+ "l185",+ "l186",+ "l187",+ "l188"+ ]+ },+ "12": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l189",+ "l190",+ "l191",+ "l192",+ "l193",+ "l194",+ "l195",+ "l196",+ "l197",+ "l198",+ "l199",+ "l200",+ "l201",+ "l202",+ "l203",+ "l204"+ ]+ },+ "13": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l205",+ "l206",+ "l207",+ "l208",+ "l209",+ "l210",+ "l211",+ "l212",+ "l213",+ "l214",+ "l215",+ "l216",+ "l217",+ "l218",+ "l219",+ "l220"+ ]+ },+ "14": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l221",+ "l222",+ "l223",+ "l224",+ "l225",+ "l226",+ "l227",+ "l228",+ "l229",+ "l230",+ "l231",+ "l232",+ "l233",+ "l234",+ "l235"+ ]+ },+ "15": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l236",+ "l237",+ "l238",+ "l239",+ "l240",+ "l241",+ "l242",+ "l243",+ "l244",+ "l245",+ "l246",+ "l247",+ "l248",+ "l249",+ "l250",+ "l251"+ ]+ },+ "16": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l252",+ "l253",+ "l254",+ "l255",+ "l256",+ "l257",+ "l258",+ "l259",+ "l260",+ "l261",+ "l262",+ "l263",+ "l264",+ "l265",+ "l266",+ "l267"+ ]+ },+ "17": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l268",+ "l269",+ "l270",+ "l271",+ "l272",+ "l273",+ "l274",+ "l275",+ "l276",+ "l277",+ "l278",+ "l279",+ "l280",+ "l281",+ "l282",+ "l283"+ ]+ },+ "18": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l284",+ "l285",+ "l286",+ "l287",+ "l288",+ "l289",+ "l290",+ "l291",+ "l292",+ "l293",+ "l294",+ "l295",+ "l296",+ "l297",+ "l298",+ "l299"+ ]+ }+ },+ {+ "0": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz"+ ]+ },+ "1": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz",+ "zzz"+ ]+ }+ },+ {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "zzz",+ "zzz"+ ]+ }+ }+ ]+ },+ {+ "name": "dict-missing-answer",+ "group_size": 16,+ "mode": "missing",+ "label": null,+ "questions": {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "criteria": {+ "l00": "about l00",+ "l01": "about l01",+ "l02": "about l02",+ "l03": "about l03",+ "l04": "about l04",+ "l05": "about l05",+ "l06": "about l06",+ "l07": "about l07",+ "l08": "about l08",+ "l09": "about l09",+ "l10": "about l10",+ "l11": "about l11",+ "l12": "about l12",+ "l13": "about l13",+ "l14": "about l14",+ "l15": "about l15",+ "l16": "about l16",+ "l17": "about l17",+ "l18": "about l18",+ "l19": "about l19",+ "l20": "about l20",+ "l21": "about l21",+ "l22": "about l22",+ "l23": "about l23",+ "l24": "about l24",+ "l25": "about l25",+ "l26": "about l26",+ "l27": "about l27",+ "l28": "about l28",+ "l29": "about l29",+ "l30": "about l30",+ "l31": "about l31",+ "l32": "about l32",+ "l33": "about l33",+ "l34": "about l34",+ "l35": "about l35",+ "l36": "about l36",+ "l37": "about l37",+ "l38": "about l38",+ "l39": "about l39"+ }+ }+ },+ "error": "KeyError",+ "message": "'0'",+ "calls": [+ {+ "0": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l00",+ "l01",+ "l02",+ "l03",+ "l04",+ "l05",+ "l06",+ "l07",+ "l08",+ "l09",+ "l10",+ "l11",+ "l12"+ ],+ "criteria": {+ "l00": "about l00",+ "l01": "about l01",+ "l02": "about l02",+ "l03": "about l03",+ "l04": "about l04",+ "l05": "about l05",+ "l06": "about l06",+ "l07": "about l07",+ "l08": "about l08",+ "l09": "about l09",+ "l10": "about l10",+ "l11": "about l11",+ "l12": "about l12"+ }+ },+ "1": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l13",+ "l14",+ "l15",+ "l16",+ "l17",+ "l18",+ "l19",+ "l20",+ "l21",+ "l22",+ "l23",+ "l24",+ "l25"+ ],+ "criteria": {+ "l13": "about l13",+ "l14": "about l14",+ "l15": "about l15",+ "l16": "about l16",+ "l17": "about l17",+ "l18": "about l18",+ "l19": "about l19",+ "l20": "about l20",+ "l21": "about l21",+ "l22": "about l22",+ "l23": "about l23",+ "l24": "about l24",+ "l25": "about l25"+ }+ },+ "2": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l26",+ "l27",+ "l28",+ "l29",+ "l30",+ "l31",+ "l32",+ "l33",+ "l34",+ "l35",+ "l36",+ "l37",+ "l38",+ "l39"+ ],+ "criteria": {+ "l26": "about l26",+ "l27": "about l27",+ "l28": "about l28",+ "l29": "about l29",+ "l30": "about l30",+ "l31": "about l31",+ "l32": "about l32",+ "l33": "about l33",+ "l34": "about l34",+ "l35": "about l35",+ "l36": "about l36",+ "l37": "about l37",+ "l38": "about l38",+ "l39": "about l39"+ }+ }+ }+ ]+ },+ {+ "name": "list-missing-answer",+ "group_size": 16,+ "mode": "missing",+ "label": null,+ "questions": {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "criteria": [+ "l00",+ "l01",+ "l02",+ "l03",+ "l04",+ "l05",+ "l06",+ "l07",+ "l08",+ "l09",+ "l10",+ "l11",+ "l12",+ "l13",+ "l14",+ "l15",+ "l16",+ "l17",+ "l18",+ "l19",+ "l20",+ "l21",+ "l22",+ "l23",+ "l24",+ "l25",+ "l26",+ "l27",+ "l28",+ "l29",+ "l30",+ "l31",+ "l32",+ "l33",+ "l34",+ "l35",+ "l36",+ "l37",+ "l38",+ "l39"+ ]+ }+ },+ "error": "KeyError",+ "message": "'0'",+ "calls": [+ {+ "0": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l00",+ "l01",+ "l02",+ "l03",+ "l04",+ "l05",+ "l06",+ "l07",+ "l08",+ "l09",+ "l10",+ "l11",+ "l12"+ ]+ },+ "1": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l13",+ "l14",+ "l15",+ "l16",+ "l17",+ "l18",+ "l19",+ "l20",+ "l21",+ "l22",+ "l23",+ "l24",+ "l25"+ ]+ },+ "2": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l26",+ "l27",+ "l28",+ "l29",+ "l30",+ "l31",+ "l32",+ "l33",+ "l34",+ "l35",+ "l36",+ "l37",+ "l38",+ "l39"+ ]+ }+ }+ ]+ },+ {+ "name": "dict-answer-without-choice",+ "group_size": 16,+ "mode": "no-choice",+ "label": null,+ "questions": {+ "q": {+ "type": "choice",+ "instructions": "pick",+ "criteria": {+ "l00": "about l00",+ "l01": "about l01",+ "l02": "about l02",+ "l03": "about l03",+ "l04": "about l04",+ "l05": "about l05",+ "l06": "about l06",+ "l07": "about l07",+ "l08": "about l08",+ "l09": "about l09",+ "l10": "about l10",+ "l11": "about l11",+ "l12": "about l12",+ "l13": "about l13",+ "l14": "about l14",+ "l15": "about l15",+ "l16": "about l16",+ "l17": "about l17",+ "l18": "about l18",+ "l19": "about l19",+ "l20": "about l20",+ "l21": "about l21",+ "l22": "about l22",+ "l23": "about l23",+ "l24": "about l24",+ "l25": "about l25",+ "l26": "about l26",+ "l27": "about l27",+ "l28": "about l28",+ "l29": "about l29",+ "l30": "about l30",+ "l31": "about l31",+ "l32": "about l32",+ "l33": "about l33",+ "l34": "about l34",+ "l35": "about l35",+ "l36": "about l36",+ "l37": "about l37",+ "l38": "about l38",+ "l39": "about l39"+ }+ }+ },+ "error": "KeyError",+ "message": "'choice'",+ "calls": [+ {+ "0": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l00",+ "l01",+ "l02",+ "l03",+ "l04",+ "l05",+ "l06",+ "l07",+ "l08",+ "l09",+ "l10",+ "l11",+ "l12"+ ],+ "criteria": {+ "l00": "about l00",+ "l01": "about l01",+ "l02": "about l02",+ "l03": "about l03",+ "l04": "about l04",+ "l05": "about l05",+ "l06": "about l06",+ "l07": "about l07",+ "l08": "about l08",+ "l09": "about l09",+ "l10": "about l10",+ "l11": "about l11",+ "l12": "about l12"+ }+ },+ "1": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l13",+ "l14",+ "l15",+ "l16",+ "l17",+ "l18",+ "l19",+ "l20",+ "l21",+ "l22",+ "l23",+ "l24",+ "l25"+ ],+ "criteria": {+ "l13": "about l13",+ "l14": "about l14",+ "l15": "about l15",+ "l16": "about l16",+ "l17": "about l17",+ "l18": "about l18",+ "l19": "about l19",+ "l20": "about l20",+ "l21": "about l21",+ "l22": "about l22",+ "l23": "about l23",+ "l24": "about l24",+ "l25": "about l25"+ }+ },+ "2": {+ "type": "choice",+ "instructions": "pick",+ "labels": [+ "l26",+ "l27",+ "l28",+ "l29",+ "l30",+ "l31",+ "l32",+ "l33",+ "l34",+ "l35",+ "l36",+ "l37",+ "l38",+ "l39"+ ],+ "criteria": {+ "l26": "about l26",+ "l27": "about l27",+ "l28": "about l28",+ "l29": "about l29",+ "l30": "about l30",+ "l31": "about l31",+ "l32": "about l32",+ "l33": "about l33",+ "l34": "about l34",+ "l35": "about l35",+ "l36": "about l36",+ "l37": "about l37",+ "l38": "about l38",+ "l39": "about l39"+ }+ }+ }+ ]+ }+ ],"refusals": [{"group_size": 1,diff --git a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Shortlist.java b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Shortlist.javaindex 670bf6e4..4ee5de48 100644--- a/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Shortlist.java+++ b/laya-java/laya-java/src/main/java/com/convaiinnovations/laya/Shortlist.java@@ -302,10 +302,14 @@ public final class Shortlist {* tournament choice's probabilities are over its finalists only. The caller's map is not* modified.*- * @throws IllegalArgumentException if {@code groupSize} is below two- * @throws IllegalStateException if a round's prediction does not answer a group with one of- * its labels; the reference fails on such a label at the next- * cut instead+ * <p>A round's answer advances as given, as in the reference: an answer from outside its group+ * is carried forward, and duplicates collapse when the question is cut.+ *+ * @throws IllegalArgumentException if {@code groupSize} is below two; if a round's prediction+ * has no choice answer for a group (the reference's+ * {@code KeyError}, raised at once); or if a question is cut+ * to a label it does not have (its {@code KeyError} at that+ * cut, in the next round or the final call)*/public static Tournament predictTournament(Predictor predictor, Object state,Map<String, Question> questions, int groupSize) {@@ -352,10 +356,10 @@ public final class Shortlist {for (int i = 0; i < groups.size(); i++) {String id = Integer.toString(i);Answer answer = answered == null ? null : answered.answer(id);- if (!(answer instanceof Answer.Choice choice)- || !groups.get(i).contains(choice.choice())) {- throw new IllegalStateException(String.format(- "round question %s of %s was not answered with one of its labels: %s",+ // Only a missing choice fails here; any label advances, as in the reference.+ if (!(answer instanceof Answer.Choice choice) || choice.choice() == null) {+ throw new IllegalArgumentException(String.format(+ "round question %s of %s has no choice answer: %s",id, PythonJson.repr(owners.get(i)), answer));}winners.computeIfAbsent(owners.get(i), key -> new ArrayList<>())@@ -409,6 +413,10 @@ public final class Shortlist {Map<String, Object> criteria = criteriaOf(question);Map<String, Object> subset = new LinkedHashMap<>();for (String label : labels) {+ if (!criteria.containsKey(label)) {+ throw new IllegalArgumentException(+ "the choice has no label " + PythonJson.repr(label) + " to cut to");+ }subset.put(label, criteria.get(label));}return Question.choice(question.instructions(), subset);diff --git a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/TournamentTest.java b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/TournamentTest.javaindex e6bb0dcf..9de2adc3 100644--- a/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/TournamentTest.java+++ b/laya-java/laya-java/src/test/java/com/convaiinnovations/laya/TournamentTest.java@@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;import java.nio.charset.StandardCharsets;import java.util.ArrayList;import java.util.LinkedHashMap;+import java.util.LinkedHashSet;import java.util.List;import java.util.Map;import org.junit.jupiter.api.DisplayName;@@ -345,19 +346,85 @@ class TournamentTest {assertEquals(List.of("small"), new ArrayList<>(out.tournament().keySet()));}+ @TestFactory+ @DisplayName("irregular round answers advance or fail exactly where the reference does")+ List<DynamicTest> irregularAnswers() {+ List<DynamicTest> tests = new ArrayList<>();+ String state = (String) fixture().get("state");+ for (Object entry : list(fixture().get("irregular"))) {+ Map<String, Object> row = map(entry);+ String name = (String) row.get("name");+ tests.add(DynamicTest.dynamicTest(name, () -> assertIrregular(name, state, row)));+ }+ assertTrue(tests.size() >= 9, "only " + tests.size() + " irregular cases");+ return tests;+ }++ /**+ * One irregular case. A Java choice always has map criteria, so a list-criteria case is held+ * to the dict outcome: duplicates collapse in what is asked, and a non-label fails at its cut.+ */+ private static void assertIrregular(String name, String state, Map<String, Object> row) {+ Map<String, Object> spec = map(map(row.get("questions")).get("q"));+ boolean listCriteria = spec.get("criteria") instanceof List;+ String mode = (String) row.get("mode");+ String label = (String) row.get("label");+ List<Map<String, Question>> calls = new ArrayList<>();+ Predictor fixed = (s, qs) -> {+ calls.add(new LinkedHashMap<>(qs));+ Map<String, Answer> answers = new LinkedHashMap<>();+ for (String id : qs.keySet()) {+ if ("label".equals(mode)) {+ answers.put(id, new Answer.Choice(label, Map.of(), 1, 1, 0));+ } else if ("no-choice".equals(mode)) {+ answers.put(id, new Answer.Noul(0.5, 1, 1, 0));+ }+ }+ return new Prediction("fixed-" + calls.size(), answers, noUsage());+ };+ Map<String, Question> questions = questions(map(row.get("questions")));+ boolean nonLabel = "label".equals(mode) && !questions.get("q").labels().contains(label);+ Object error = row.get("error");+ if (listCriteria && nonLabel) {+ assertEquals(null, error, name + ": the reference accepts a non-label in a list");+ error = "KeyError";+ }+ List<Object> recorded = list(row.get("calls"));+ if (error != null) {+ assertEquals("KeyError", error, name);+ IllegalArgumentException failed = assertThrows(IllegalArgumentException.class,+ () -> Shortlist.predictTournament(fixed, state, questions, 16));+ assertTrue(failed.getMessage() != null && !failed.getMessage().isEmpty());+ if (!(listCriteria && nonLabel)) {+ assertEquals(recorded.size(), calls.size(), name + ": calls before the failure");+ }+ return;+ }+ Shortlist.Tournament result = Shortlist.predictTournament(fixed, state, questions, 16);+ assertEquals(recorded.size(), calls.size(), name + ": calls");+ for (int c = 0; c < recorded.size(); c++) {+ Map<String, Object> asked = map(recorded.get(c));+ assertEquals(new ArrayList<>(asked.keySet()), new ArrayList<>(calls.get(c).keySet()));+ for (Map.Entry<String, Object> q : asked.entrySet()) {+ List<String> labels = new ArrayList<>(new LinkedHashSet<>(+ strings(map(q.getValue()).get("labels"))));+ assertEquals(labels, calls.get(c).get(q.getKey()).labels(),+ name + " call " + c + "." + q.getKey());+ }+ }+ Map<String, Object> want = map(map(row.get("tournament")).get("q"));+ Shortlist.Bracket got = result.tournament().get("q");+ assertEquals(strings(want.get("labels")), got.labels(), name + ": bracket labels");+ assertEquals(((Number) want.get("rounds")).intValue(), got.rounds());+ assertEquals(map(row.get("answers")).get("q"),+ ((Answer.Choice) result.prediction().answer("q")).choice());+ }+@Test- @DisplayName("a round answer outside its group, or missing, is refused")- void badRoundAnswerIsRefused() {- Predictor outside = (s, qs) -> new Prediction("stub", Map.of("0",- new Answer.Choice("l0039", Map.of(), 1, 1, 0)), noUsage());- IllegalStateException wrong = assertThrows(IllegalStateException.class,- () -> Shortlist.predictTournament(outside, "s", oneChoice(40), 16));- assertTrue(wrong.getMessage().contains("round question 0 of 'q'"), wrong.getMessage());- Predictor silent = (s, qs) -> new Prediction("stub", Map.of(), noUsage());- assertThrows(IllegalStateException.class,- () -> Shortlist.predictTournament(silent, "s", oneChoice(40), 16));+ @DisplayName("a null round prediction is refused like a missing answer")+ void nullRoundPredictionIsRefused() {Predictor nothing = (s, qs) -> null;- assertThrows(IllegalStateException.class,+ assertThrows(IllegalArgumentException.class,() -> Shortlist.predictTournament(nothing, "s", oneChoice(40), 16));assertThrows(IllegalArgumentException.class,() -> Shortlist.predictTournament(nothing, "s", oneChoice(3), 16),diff --git a/laya-java/scripts/gen_fixtures.py b/laya-java/scripts/gen_fixtures.pyindex 429bc4fa..e3d6f1d1 100644--- a/laya-java/scripts/gen_fixtures.py+++ b/laya-java/scripts/gen_fixtures.py@@ -1128,14 +1128,16 @@ class _TournamentRecorder:def _asked(questions):- """A recorded call: each question's type, instructions, labels, and dict criteria verbatim."""- from laya import shortlist as mod+ """A recorded call: each question's type, instructions, labels, and dict criteria verbatim.+ Labels are read without validation, so a call the reference really made with a duplicated+ list label is recorded as made.+ """out = {}for qid, qdef in questions.items():entry = {"type": qdef.get("type"), "instructions": qdef.get("instructions")}if qdef.get("type") == "choice":- entry["labels"] = [key for key, _value in mod._criteria_items(qdef["criteria"])]+ entry["labels"] = list(qdef["criteria"])if isinstance(qdef["criteria"], dict):entry["criteria"] = qdef["criteria"]out[qid] = entry@@ -1226,6 +1228,54 @@ def tournament():"caller_questions_unmutated": unchanged,})+ # A model that answers outside the group, with a non-label, or not at all. The reference+ # advances whatever a round answered and fails only on a missing answer (at once) or when a+ # dict-criteria question is cut to a label it lacks.+ class Fixed:+ def __init__(self, mode, label=None):+ self.mode, self.label, self.calls = mode, label, []++ def predict(self, state, questions, **kwargs):+ self.calls.append({qid: dict(qdef) for qid, qdef in questions.items()})+ if self.mode == "missing":+ answers = {}+ elif self.mode == "no-choice":+ answers = {qid: {"type": "noul", "noul": 0.5} for qid in questions}+ else:+ answers = {qid: {"type": "choice", "choice": self.label} for qid in questions}+ return {"model": "fixed-%d" % len(self.calls), "answers": answers}++ def forty(as_list, n=40):+ labels = ["l%02d" % i if n <= 100 else "l%03d" % i for i in range(n)]+ return labels if as_list else {label: "about " + label for label in labels}++ irregular = []+ for name, criteria, mode, label in [+ ("dict-out-of-group", forty(False), "label", "l39"),+ ("list-out-of-group", forty(True), "label", "l39"),+ ("dict-non-label", forty(False), "label", "zzz"),+ ("list-non-label", forty(True), "label", "zzz"),+ ("dict-non-label-next-round", forty(False, 300), "label", "zzz"),+ ("list-non-label-two-rounds", forty(True, 300), "label", "zzz"),+ ("dict-missing-answer", forty(False), "missing", None),+ ("list-missing-answer", forty(True), "missing", None),+ ("dict-answer-without-choice", forty(False), "no-choice", None),+ ]:+ questions = {"q": {"type": "choice", "instructions": "pick", "criteria": criteria}}+ model = Fixed(mode, label)+ row = {"name": name, "group_size": 16, "mode": mode, "label": label,+ "questions": questions}+ try:+ result = mod.predict_tournament(model, state, questions, group_size=16)+ row["error"] = None+ row["answers"] = {qid: a["choice"] for qid, a in result["answers"].items()}+ row["tournament"] = result["tournament"]+ except Exception as failure:+ row["error"] = type(failure).__name__+ row["message"] = str(failure)+ row["calls"] = [_asked(call) for call in model.calls]+ irregular.append(row)+refusals = []for bad in (1, 0, -3):try:@@ -1241,6 +1291,7 @@ def tournament():"rank_probe": {label: str(_tournament_rank(label)) for label in ["", "a", "l0001", "注文"]},"splits": splits,"cases": cases,+ "irregular": irregular,"refusals": refusals,}