Ears for coding agents: let AI hear, review and make sound. CPU-only, open source.
{
"n_eval": 219,
"n_ref": 148,
"n_vocab": 322,
"chosen": {
"pad": "repeatpad",
"templates": "t5",
"method": "both",
"all": {
"cat_map": 0.7775335919864723,
"cat_p5": 0.85,
"cat_p10": 0.85,
"class_map": 0.6424589753467237,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5799086757990868,
"tag_mrr": 0.712406871471145,
"pooled_auc": 0.9656601066427873,
"pooled_auc_strict": 0.9508040386214509,
"spec_top1": 0.7916666666666666,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.777586606292167,
"cat_p5": 0.85,
"cat_p10": 0.75,
"class_map": 0.6779892152094823,
"class_top3": 0.8870967741935484,
"tag_acc": 0.6008064516129032,
"tag_mrr": 0.7313790616536111,
"pooled_auc": 0.9731773544405689,
"pooled_auc_strict": 0.9595833926453143,
"spec_top1": 0.9,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7932570388183038,
"cat_p5": 0.825,
"cat_p10": 0.7125,
"class_map": 0.6390066424489291,
"class_top3": 0.8793103448275862,
"tag_acc": 0.5526315789473685,
"tag_mrr": 0.6876431706014001,
"pooled_auc": 0.9558856317750858,
"pooled_auc_strict": 0.9393954636620294,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
},
"detail": {
"category": {
"map": 0.7775335919864723,
"p5": 0.85,
"p10": 0.85,
"rows": [
{
"cat": "ambience",
"ap": 0.9464513817222796,
"p5": 1.0,
"p10": 1.0
},
{
"cat": "foley",
"ap": 0.23496220076980903,
"p5": 0.0,
"p10": 0.2
},
{
"cat": "garden",
"ap": 0.7607732443514181,
"p5": 1.0,
"p10": 0.9
},
{
"cat": "keyboard",
"ap": 0.9541322251479479,
"p5": 1.0,
"p10": 1.0
},
{
"cat": "magic",
"ap": 1.0,
"p5": 1.0,
"p10": 1.0
},
{
"cat": "motion",
"ap": 0.671879351245011,
"p5": 1.0,
"p10": 1.0
},
{
"cat": "music",
"ap": 1.0,
"p5": 1.0,
"p10": 0.8
},
{
"cat": "toy",
"ap": 0.6520703326553127,
"p5": 0.8,
"p10": 0.9
}
]
},
"class": {
"map": 0.6424589753467237,
"top3": 0.8857142857142857,
"n_queries": 70
},
"specific": {
"top1": 0.7916666666666666,
"top3": 0.9583333333333334,
"map": 0.7557323637715152,
"rows": [
{
"q": "a deep thock of a mechanical keyboard key",
"top1": true,
"top3": true,
"ap": 0.65227130473939,
"best": "fs-789628_keychron-linear-individual-keys.ogg"
},
{
"q": "a whoosh",
"top1": true,
"top3": true,
"ap": 0.960530045351474,
"best": "mk-1489_air-woosh.wav"
},
{
"q": "a glass ting",
"top1": true,
"top3": true,
"ap": 0.6752136752136751,
"best": "fs-549899_small-wine-glass-tap.ogg"
},
{
"q": "footsteps in grass",
"top1": true,
"top3": true,
"ap": 0.8168168880455408,
"best": "mk-1921_walking-on-grass.wav"
},
{
"q": "rain on a window",
"top1": true,
"top3": true,
"ap": 0.9761904761904762,
"best": "fs-346642_rain-on-windows-interior.ogg"
},
{
"q": "a cork pop",
"top1": false,
"top3": false,
"ap": 0.25,
"best": "fs-181907_mouth-plops-c214.ogg"
},
{
"q": "a pop",
"top1": true,
"top3": true,
"ap": 0.8692956349206349,
"best": "fs-181907_mouth-plops-c214.ogg"
},
{
"q": "a light switch click",
"top1": false,
"top3": true,
"ap": 0.210921912624699,
"best": "fs-151778_clip-in.ogg"
},
{
"q": "lego bricks clicking",
"top1": false,
"top3": true,
"ap": 0.43934645601312267,
"best": "fs-326470_dominos-falling.ogg"
},
{
"q": "dominoes falling",
"top1": true,
"top3": true,
"ap": 1.0,
"best": "fs-873306_various-domino-falling.ogg"
},
{
"q": "a celesta note",
"top1": true,
"top3": true,
"ap": 1.0,
"best": "fs-410664_celesta-B5.ogg"
},
{
"q": "a kalimba note",
"top1": true,
"top3": true,
"ap": 0.8761904761904761,
"best": "fs-681791_kalimba-D-high.ogg"
},
{
"q": "birds singing in the morning",
"top1": true,
"top3": true,
"ap": 1.0,
"best": "mk-2472_morning-birds.wav"
},
{
"q": "an object dropping into water",
"top1": true,
"top3": true,
"ap": 0.8441043083900226,
"best": "fs-272392_object-drops-in-water.ogg"
},
{
"q": "a cymbal swell",
"top1": true,
"top3": true,
"ap": 0.8928571428571428,
"best": "fs-346023_violin-bow-on-cymbal.ogg"
},
{
"q": "typing on a keyboard",
"top1": true,
"top3": true,
"ap": 0.9166666666666666,
"best": "fs-789627_keychron-linear-typing-66wpm.ogg"
},
{
"q": "a creaking wooden sound",
"top1": true,
"top3": true,
"ap": 0.649133022774327,
"best": "fs-123712_creaking-wood-screws.ogg"
},
{
"q": "digging in soil",
"top1": true,
"top3": true,
"ap": 0.6083333333333334,
"best": "fs-696531_trowel-soft-soil.ogg"
},
{
"q": "an apple falling on grass",
"top1": false,
"top3": true,
"ap": 0.6472222222222223,
"best": "fs-559082_high-grass-movement.ogg"
},
{
"q": "a finger squeaking on glass",
"top1": true,
"top3": true,
"ap": 0.7784090909090909,
"best": "fs-119916_glass-squeaking.ogg"
},
{
"q": "a computer hum",
"top1": true,
"top3": true,
"ap": 0.7333333333333334,
"best": "fs-381258_computer-hum-gentle.ogg"
},
{
"q": "a camera shutter",
"top1": true,
"top3": true,
"ap": 1.0,
"best": "fs-270435_shutter-click-canon.ogg"
},
{
"q": "a suction cup pop",
"top1": false,
"top3": true,
"ap": 0.34074074074074073,
"best": "fs-181907_mouth-plops-c214.ogg"
},
{
"q": "a deep cinematic hit",
"top1": true,
"top3": true,
"ap": 1.0,
"best": "mk-563_drum-deep-impact.wav"
}
]
},
"tagging": {
"acc": 0.5799086757990868,
"mrr": 0.712406871471145
},
"pooled_auc": 0.9656601066427873,
"pooled_auc_strict": 0.9508040386214509
}
},
"variants": [
{
"pad": "repeatpad",
"templates": "t1",
"method": "raw",
"all": {
"cat_map": 0.6980522113170913,
"cat_p5": 0.8500000000000001,
"cat_p10": 0.7875,
"class_map": 0.5970493925255879,
"class_top3": 0.8428571428571429,
"tag_acc": 0.5525114155251142,
"tag_mrr": 0.6852196436140946,
"pooled_auc": 0.9435436000759796,
"pooled_auc_strict": 0.9273817348182524,
"spec_top1": 0.7083333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.6856773137576235,
"cat_p5": 0.75,
"cat_p10": 0.6625,
"class_map": 0.6238167679338643,
"class_top3": 0.8225806451612904,
"tag_acc": 0.5806451612903226,
"tag_mrr": 0.7081935069387318,
"pooled_auc": 0.9537306144352026,
"pooled_auc_strict": 0.9380697508896797,
"spec_top1": 0.7,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.697124334270687,
"cat_p5": 0.75,
"cat_p10": 0.6499999999999999,
"class_map": 0.5876740068290082,
"class_top3": 0.7758620689655172,
"tag_acc": 0.5157894736842106,
"tag_mrr": 0.6552327062219364,
"pooled_auc": 0.9308220171927477,
"pooled_auc_strict": 0.9139352508335569,
"spec_top1": 0.5714285714285714,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "repeatpad",
"templates": "t1",
"method": "sound",
"all": {
"cat_map": 0.7439785814201825,
"cat_p5": 0.8500000000000001,
"cat_p10": 0.8,
"class_map": 0.6094180530398979,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5525114155251142,
"tag_mrr": 0.6852196436140946,
"pooled_auc": 0.9564496160316943,
"pooled_auc_strict": 0.9402166676955068,
"spec_top1": 0.875,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7456934125575034,
"cat_p5": 0.825,
"cat_p10": 0.7250000000000001,
"class_map": 0.6627912840247255,
"class_top3": 0.8709677419354839,
"tag_acc": 0.5806451612903226,
"tag_mrr": 0.7081935069387318,
"pooled_auc": 0.965224040783472,
"pooled_auc_strict": 0.9494149466192171,
"spec_top1": 0.9,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.759221115646978,
"cat_p5": 0.8,
"cat_p10": 0.7,
"class_map": 0.6008358348396999,
"class_top3": 0.8103448275862069,
"tag_acc": 0.5157894736842106,
"tag_mrr": 0.6552327062219364,
"pooled_auc": 0.9450525727188112,
"pooled_auc_strict": 0.9283196962148214,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "repeatpad",
"templates": "t1",
"method": "center",
"all": {
"cat_map": 0.6980522113170913,
"cat_p5": 0.8500000000000001,
"cat_p10": 0.7875,
"class_map": 0.5970493925255879,
"class_top3": 0.8428571428571429,
"tag_acc": 0.5684931506849316,
"tag_mrr": 0.7059231884033613,
"pooled_auc": 0.9489340809182677,
"pooled_auc_strict": 0.9332191925904139,
"spec_top1": 0.7083333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.6856773137576235,
"cat_p5": 0.75,
"cat_p10": 0.6625,
"class_map": 0.6238167679338643,
"class_top3": 0.8225806451612904,
"tag_acc": 0.6008064516129032,
"tag_mrr": 0.730970754156238,
"pooled_auc": 0.9599833646364369,
"pooled_auc_strict": 0.9452052194543298,
"spec_top1": 0.7,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.697124334270687,
"cat_p5": 0.75,
"cat_p10": 0.6499999999999999,
"class_map": 0.5876740068290082,
"class_top3": 0.7758620689655172,
"tag_acc": 0.5263157894736842,
"tag_mrr": 0.6732295236311856,
"pooled_auc": 0.9349122844270794,
"pooled_auc_strict": 0.9179002350566563,
"spec_top1": 0.5714285714285714,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "repeatpad",
"templates": "t1",
"method": "both",
"all": {
"cat_map": 0.7610068662109295,
"cat_p5": 0.85,
"cat_p10": 0.825,
"class_map": 0.6130333886405536,
"class_top3": 0.9,
"tag_acc": 0.5684931506849316,
"tag_mrr": 0.7059231884033613,
"pooled_auc": 0.9602390982850321,
"pooled_auc_strict": 0.9444559432467534,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7534607877045285,
"cat_p5": 0.825,
"cat_p10": 0.7374999999999999,
"class_map": 0.6746554502564531,
"class_top3": 0.8709677419354839,
"tag_acc": 0.6008064516129032,
"tag_mrr": 0.730970754156238,
"pooled_auc": 0.9693265360880065,
"pooled_auc_strict": 0.9548673784104389,
"spec_top1": 0.9,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7673374351307555,
"cat_p5": 0.8,
"cat_p10": 0.725,
"class_map": 0.6112840868653101,
"class_top3": 0.8275862068965517,
"tag_acc": 0.5263157894736842,
"tag_mrr": 0.6732295236311856,
"pooled_auc": 0.9484651101885965,
"pooled_auc_strict": 0.9309656804506956,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "repeatpad",
"templates": "t1",
"method": "term_sd",
"all": {
"cat_map": 0.6980522113170913,
"cat_p5": 0.8500000000000001,
"cat_p10": 0.7875,
"class_map": 0.5970493925255879,
"class_top3": 0.8428571428571429,
"tag_acc": 0.5365296803652968,
"tag_mrr": 0.6865289901992316,
"pooled_auc": 0.946785452621296,
"pooled_auc_strict": 0.9314520082355633,
"spec_top1": 0.7083333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.6856773137576235,
"cat_p5": 0.75,
"cat_p10": 0.6625,
"class_map": 0.6238167679338643,
"class_top3": 0.8225806451612904,
"tag_acc": 0.5806451612903226,
"tag_mrr": 0.7158915345407281,
"pooled_auc": 0.9578196941239603,
"pooled_auc_strict": 0.9434927639383155,
"spec_top1": 0.7,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.697124334270687,
"cat_p5": 0.75,
"cat_p10": 0.6499999999999999,
"class_map": 0.5876740068290082,
"class_top3": 0.7758620689655172,
"tag_acc": 0.4789473684210526,
"tag_mrr": 0.648203142848226,
"pooled_auc": 0.9328622846040324,
"pooled_auc_strict": 0.9161987889472272,
"spec_top1": 0.5714285714285714,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "repeatpad",
"templates": "t1",
"method": "both_sd",
"all": {
"cat_map": 0.7586182144969197,
"cat_p5": 0.825,
"cat_p10": 0.8125,
"class_map": 0.595603722072477,
"class_top3": 0.8714285714285714,
"tag_acc": 0.5365296803652968,
"tag_mrr": 0.6865289901992316,
"pooled_auc": 0.9574583808205797,
"pooled_auc_strict": 0.9412123731954445,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7479252739679811,
"cat_p5": 0.8,
"cat_p10": 0.725,
"class_map": 0.6719452435180059,
"class_top3": 0.8548387096774194,
"tag_acc": 0.5806451612903226,
"tag_mrr": 0.7158915345407281,
"pooled_auc": 0.9660998121813791,
"pooled_auc_strict": 0.9513580071174377,
"spec_top1": 0.85,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7880552005178514,
"cat_p5": 0.8,
"cat_p10": 0.7,
"class_map": 0.6064812958523895,
"class_top3": 0.8448275862068966,
"tag_acc": 0.4789473684210526,
"tag_mrr": 0.648203142848226,
"pooled_auc": 0.9462222316597124,
"pooled_auc_strict": 0.9280394486388431,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "repeatpad",
"templates": "t3",
"method": "raw",
"all": {
"cat_map": 0.718118751880866,
"cat_p5": 0.825,
"cat_p10": 0.7875,
"class_map": 0.6273662801196275,
"class_top3": 0.8857142857142857,
"tag_acc": 0.589041095890411,
"tag_mrr": 0.7097763961710694,
"pooled_auc": 0.9475559331922284,
"pooled_auc_strict": 0.9320818702620396,
"spec_top1": 0.875,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7153805164806042,
"cat_p5": 0.825,
"cat_p10": 0.675,
"class_map": 0.6480120430856741,
"class_top3": 0.8064516129032258,
"tag_acc": 0.625,
"tag_mrr": 0.7339496162100214,
"pooled_auc": 0.9563938824792058,
"pooled_auc_strict": 0.9414861209964412,
"spec_top1": 0.85,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7204871594793816,
"cat_p5": 0.75,
"cat_p10": 0.675,
"class_map": 0.6354754574664218,
"class_top3": 0.8620689655172413,
"tag_acc": 0.5421052631578948,
"tag_mrr": 0.6782239826465425,
"pooled_auc": 0.9363978043679064,
"pooled_auc_strict": 0.9201422156644822,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9523809523809523
}
},
{
"pad": "repeatpad",
"templates": "t3",
"method": "sound",
"all": {
"cat_map": 0.7665978592943665,
"cat_p5": 0.85,
"cat_p10": 0.8375,
"class_map": 0.6414165364178294,
"class_top3": 0.9142857142857143,
"tag_acc": 0.589041095890411,
"tag_mrr": 0.7097763961710694,
"pooled_auc": 0.9601956813741452,
"pooled_auc_strict": 0.9447617206081534,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7634370504834116,
"cat_p5": 0.875,
"cat_p10": 0.7375,
"class_map": 0.6690742346670663,
"class_top3": 0.8870967741935484,
"tag_acc": 0.625,
"tag_mrr": 0.7339496162100214,
"pooled_auc": 0.9687255165012074,
"pooled_auc_strict": 0.9539074733096086,
"spec_top1": 0.85,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7911520237903075,
"cat_p5": 0.8,
"cat_p10": 0.7125,
"class_map": 0.638673491556023,
"class_top3": 0.8620689655172413,
"tag_acc": 0.5421052631578948,
"tag_mrr": 0.6782239826465425,
"pooled_auc": 0.9492304316590046,
"pooled_auc_strict": 0.9329641410850931,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "repeatpad",
"templates": "t3",
"method": "center",
"all": {
"cat_map": 0.718118751880866,
"cat_p5": 0.825,
"cat_p10": 0.7875,
"class_map": 0.6273662801196275,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5707762557077626,
"tag_mrr": 0.7029040570512325,
"pooled_auc": 0.9517398987843265,
"pooled_auc_strict": 0.9370941376685028,
"spec_top1": 0.875,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7153805164806042,
"cat_p5": 0.825,
"cat_p10": 0.675,
"class_map": 0.6480120430856741,
"class_top3": 0.8064516129032258,
"tag_acc": 0.6088709677419355,
"tag_mrr": 0.7304551820728292,
"pooled_auc": 0.9609621679635095,
"pooled_auc_strict": 0.9471051008303677,
"spec_top1": 0.85,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7204871594793816,
"cat_p5": 0.75,
"cat_p10": 0.675,
"class_map": 0.6354754574664218,
"class_top3": 0.8620689655172413,
"tag_acc": 0.5210526315789473,
"tag_mrr": 0.6669425886019905,
"pooled_auc": 0.9400527673670465,
"pooled_auc_strict": 0.9243419371734437,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9523809523809523
}
},
{
"pad": "repeatpad",
"templates": "t3",
"method": "both",
"all": {
"cat_map": 0.7695786878859658,
"cat_p5": 0.85,
"cat_p10": 0.825,
"class_map": 0.6427246569683582,
"class_top3": 0.9,
"tag_acc": 0.5707762557077626,
"tag_mrr": 0.7029040570512325,
"pooled_auc": 0.9637309372625638,
"pooled_auc_strict": 0.9486622353913768,
"spec_top1": 0.8333333333333334,
"spec_top3": 1.0
},
"dev": {
"cat_map": 0.7662596317937798,
"cat_p5": 0.825,
"cat_p10": 0.7625,
"class_map": 0.6750202686222422,
"class_top3": 0.8709677419354839,
"tag_acc": 0.6088709677419355,
"tag_mrr": 0.7304551820728292,
"pooled_auc": 0.9718663804668635,
"pooled_auc_strict": 0.9581542111506525,
"spec_top1": 0.9,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7937494731754101,
"cat_p5": 0.8,
"cat_p10": 0.7250000000000001,
"class_map": 0.6407116440892073,
"class_top3": 0.8620689655172413,
"tag_acc": 0.5210526315789473,
"tag_mrr": 0.6669425886019905,
"pooled_auc": 0.9530924295992723,
"pooled_auc_strict": 0.9362688268884375,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "repeatpad",
"templates": "t3",
"method": "term_sd",
"all": {
"cat_map": 0.718118751880866,
"cat_p5": 0.825,
"cat_p10": 0.7875,
"class_map": 0.6273662801196275,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5593607305936074,
"tag_mrr": 0.6987727253599382,
"pooled_auc": 0.9498845042331489,
"pooled_auc_strict": 0.9355914257667582,
"spec_top1": 0.875,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7153805164806042,
"cat_p5": 0.825,
"cat_p10": 0.675,
"class_map": 0.6480120430856741,
"class_top3": 0.8064516129032258,
"tag_acc": 0.5846774193548387,
"tag_mrr": 0.7193196458691004,
"pooled_auc": 0.9590576871478401,
"pooled_auc_strict": 0.9455406880189798,
"spec_top1": 0.85,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7204871594793816,
"cat_p5": 0.75,
"cat_p10": 0.675,
"class_map": 0.6354754574664218,
"class_top3": 0.8620689655172413,
"tag_acc": 0.5263157894736842,
"tag_mrr": 0.6719535870111372,
"pooled_auc": 0.9382903160733435,
"pooled_auc_strict": 0.9229582646686851,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9523809523809523
}
},
{
"pad": "repeatpad",
"templates": "t3",
"method": "both_sd",
"all": {
"cat_map": 0.7641445315624935,
"cat_p5": 0.85,
"cat_p10": 0.825,
"class_map": 0.6208798291934929,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5593607305936074,
"tag_mrr": 0.6987727253599382,
"pooled_auc": 0.9621942160534028,
"pooled_auc_strict": 0.9466517605876735,
"spec_top1": 0.875,
"spec_top3": 1.0
},
"dev": {
"cat_map": 0.7522572784268877,
"cat_p5": 0.825,
"cat_p10": 0.7374999999999999,
"class_map": 0.6733560017280501,
"class_top3": 0.8870967741935484,
"tag_acc": 0.5846774193548387,
"tag_mrr": 0.7193196458691004,
"pooled_auc": 0.9696361685001341,
"pooled_auc_strict": 0.9554733096085409,
"spec_top1": 0.9,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7919940719791706,
"cat_p5": 0.825,
"cat_p10": 0.7000000000000001,
"class_map": 0.6408222726285296,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5263157894736842,
"tag_mrr": 0.6719535870111372,
"pooled_auc": 0.9524810571876515,
"pooled_auc_strict": 0.9351709909426539,
"spec_top1": 0.9047619047619048,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "repeatpad",
"templates": "t5",
"method": "raw",
"all": {
"cat_map": 0.7277025148739027,
"cat_p5": 0.8500000000000001,
"cat_p10": 0.825,
"class_map": 0.627087687258443,
"class_top3": 0.8857142857142857,
"tag_acc": 0.593607305936073,
"tag_mrr": 0.7149215845506086,
"pooled_auc": 0.9509806116357321,
"pooled_auc_strict": 0.9355483052580997,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7229405850544317,
"cat_p5": 0.85,
"cat_p10": 0.6875,
"class_map": 0.6431871490679331,
"class_top3": 0.8064516129032258,
"tag_acc": 0.6209677419354839,
"tag_mrr": 0.7338462142037805,
"pooled_auc": 0.9592218942849476,
"pooled_auc_strict": 0.9444052194543298,
"spec_top1": 0.85,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7199503932575724,
"cat_p5": 0.75,
"cat_p10": 0.675,
"class_map": 0.647772223495713,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6902199626875211,
"pooled_auc": 0.9405800871316282,
"pooled_auc_strict": 0.9243786647759935,
"spec_top1": 0.7142857142857143,
"spec_top3": 0.9523809523809523
}
},
{
"pad": "repeatpad",
"templates": "t5",
"method": "sound",
"all": {
"cat_map": 0.7702130140324235,
"cat_p5": 0.85,
"cat_p10": 0.8625,
"class_map": 0.649617613261983,
"class_top3": 0.9,
"tag_acc": 0.593607305936073,
"tag_mrr": 0.7149215845506086,
"pooled_auc": 0.9633435770107457,
"pooled_auc_strict": 0.9481022739789366,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7706841705015869,
"cat_p5": 0.8999999999999999,
"cat_p10": 0.7375,
"class_map": 0.6757453275548259,
"class_top3": 0.9032258064516129,
"tag_acc": 0.6209677419354839,
"tag_mrr": 0.7338462142037805,
"pooled_auc": 0.9712261872819963,
"pooled_auc_strict": 0.9565115065243179,
"spec_top1": 0.85,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7699222515537358,
"cat_p5": 0.8,
"cat_p10": 0.7125,
"class_map": 0.6454645266686625,
"class_top3": 0.8620689655172413,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6902199626875211,
"pooled_auc": 0.9532366462462972,
"pooled_auc_strict": 0.9373035871689726,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "repeatpad",
"templates": "t5",
"method": "center",
"all": {
"cat_map": 0.7277025148739027,
"cat_p5": 0.8500000000000001,
"cat_p10": 0.825,
"class_map": 0.627087687258443,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5799086757990868,
"tag_mrr": 0.712406871471145,
"pooled_auc": 0.9537319887658743,
"pooled_auc_strict": 0.9391610473835383,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7229405850544317,
"cat_p5": 0.85,
"cat_p10": 0.6875,
"class_map": 0.6431871490679331,
"class_top3": 0.8064516129032258,
"tag_acc": 0.6008064516129032,
"tag_mrr": 0.7313790616536111,
"pooled_auc": 0.9622554333243896,
"pooled_auc_strict": 0.9485385527876631,
"spec_top1": 0.85,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7199503932575724,
"cat_p5": 0.75,
"cat_p10": 0.675,
"class_map": 0.647772223495713,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5526315789473685,
"tag_mrr": 0.6876431706014001,
"pooled_auc": 0.9428848991899095,
"pooled_auc_strict": 0.9272050933200475,
"spec_top1": 0.7142857142857143,
"spec_top3": 0.9523809523809523
}
},
{
"pad": "repeatpad",
"templates": "t5",
"method": "both",
"all": {
"cat_map": 0.7775335919864723,
"cat_p5": 0.85,
"cat_p10": 0.85,
"class_map": 0.6424589753467237,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5799086757990868,
"tag_mrr": 0.712406871471145,
"pooled_auc": 0.9656601066427873,
"pooled_auc_strict": 0.9508040386214509,
"spec_top1": 0.7916666666666666,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.777586606292167,
"cat_p5": 0.85,
"cat_p10": 0.75,
"class_map": 0.6779892152094823,
"class_top3": 0.8870967741935484,
"tag_acc": 0.6008064516129032,
"tag_mrr": 0.7313790616536111,
"pooled_auc": 0.9731773544405689,
"pooled_auc_strict": 0.9595833926453143,
"spec_top1": 0.9,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7932570388183038,
"cat_p5": 0.825,
"cat_p10": 0.7125,
"class_map": 0.6390066424489291,
"class_top3": 0.8793103448275862,
"tag_acc": 0.5526315789473685,
"tag_mrr": 0.6876431706014001,
"pooled_auc": 0.9558856317750858,
"pooled_auc_strict": 0.9393954636620294,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "repeatpad",
"templates": "t5",
"method": "term_sd",
"all": {
"cat_map": 0.7277025148739027,
"cat_p5": 0.8500000000000001,
"cat_p10": 0.825,
"class_map": 0.627087687258443,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5776255707762558,
"tag_mrr": 0.7099758184972561,
"pooled_auc": 0.9517821285140562,
"pooled_auc_strict": 0.9377659098033947,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7229405850544317,
"cat_p5": 0.85,
"cat_p10": 0.6875,
"class_map": 0.6431871490679331,
"class_top3": 0.8064516129032258,
"tag_acc": 0.592741935483871,
"tag_mrr": 0.7242494221728093,
"pooled_auc": 0.9603209015293802,
"pooled_auc_strict": 0.9471311981020166,
"spec_top1": 0.85,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7199503932575724,
"cat_p5": 0.75,
"cat_p10": 0.675,
"class_map": 0.647772223495713,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6913450094891653,
"pooled_auc": 0.9409676140727135,
"pooled_auc_strict": 0.9258701248099745,
"spec_top1": 0.7142857142857143,
"spec_top3": 0.9523809523809523
}
},
{
"pad": "repeatpad",
"templates": "t5",
"method": "both_sd",
"all": {
"cat_map": 0.7684321393745663,
"cat_p5": 0.85,
"cat_p10": 0.8375,
"class_map": 0.6294999047685299,
"class_top3": 0.9,
"tag_acc": 0.5776255707762558,
"tag_mrr": 0.7099758184972561,
"pooled_auc": 0.9640507977857375,
"pooled_auc_strict": 0.9488335070257682,
"spec_top1": 0.7916666666666666,
"spec_top3": 1.0
},
"dev": {
"cat_map": 0.7597971337607392,
"cat_p5": 0.825,
"cat_p10": 0.7374999999999999,
"class_map": 0.6736543996724086,
"class_top3": 0.9032258064516129,
"tag_acc": 0.592741935483871,
"tag_mrr": 0.7242494221728093,
"pooled_auc": 0.9708870405151596,
"pooled_auc_strict": 0.9569504151838671,
"spec_top1": 0.9,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.795543349737912,
"cat_p5": 0.825,
"cat_p10": 0.7124999999999999,
"class_map": 0.6447098756668317,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6913450094891653,
"pooled_auc": 0.9551512770693762,
"pooled_auc_strict": 0.9382640938182655,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "zero",
"templates": "t1",
"method": "raw",
"all": {
"cat_map": 0.6966951514439714,
"cat_p5": 0.9,
"cat_p10": 0.75,
"class_map": 0.6103205175324865,
"class_top3": 0.8428571428571429,
"tag_acc": 0.5662100456621004,
"tag_mrr": 0.688933104494091,
"pooled_auc": 0.9470330646911972,
"pooled_auc_strict": 0.9317642914982698,
"spec_top1": 0.7083333333333334,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.6999603235345861,
"cat_p5": 0.775,
"cat_p10": 0.6375000000000001,
"class_map": 0.6421799826780327,
"class_top3": 0.8225806451612904,
"tag_acc": 0.592741935483871,
"tag_mrr": 0.704020998923095,
"pooled_auc": 0.9547206868795278,
"pooled_auc_strict": 0.9396664294187426,
"spec_top1": 0.6,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.6923689595029523,
"cat_p5": 0.7,
"cat_p10": 0.6375,
"class_map": 0.6342054217314276,
"class_top3": 0.7931034482758621,
"tag_acc": 0.531578947368421,
"tag_mrr": 0.6692394317657068,
"pooled_auc": 0.9374489048382473,
"pooled_auc_strict": 0.9218795909503187,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "zero",
"templates": "t1",
"method": "sound",
"all": {
"cat_map": 0.7296165061572158,
"cat_p5": 0.8500000000000001,
"cat_p10": 0.825,
"class_map": 0.5990407658743546,
"class_top3": 0.8571428571428571,
"tag_acc": 0.5662100456621004,
"tag_mrr": 0.688933104494091,
"pooled_auc": 0.9525578666015413,
"pooled_auc_strict": 0.9376875363876573,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7343754496700743,
"cat_p5": 0.825,
"cat_p10": 0.6875,
"class_map": 0.6395514066218183,
"class_top3": 0.8387096774193549,
"tag_acc": 0.592741935483871,
"tag_mrr": 0.704020998923095,
"pooled_auc": 0.9570480279044808,
"pooled_auc_strict": 0.9420365361803085,
"spec_top1": 0.85,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7421549661065795,
"cat_p5": 0.775,
"cat_p10": 0.675,
"class_map": 0.6013351535309124,
"class_top3": 0.7758620689655172,
"tag_acc": 0.531578947368421,
"tag_mrr": 0.6692394317657068,
"pooled_auc": 0.9466831940713687,
"pooled_auc_strict": 0.9319892627652372,
"spec_top1": 0.7142857142857143,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "zero",
"templates": "t1",
"method": "center",
"all": {
"cat_map": 0.6966951514439714,
"cat_p5": 0.9,
"cat_p10": 0.75,
"class_map": 0.6103205175324865,
"class_top3": 0.8428571428571429,
"tag_acc": 0.5684931506849316,
"tag_mrr": 0.700051267624108,
"pooled_auc": 0.9540455741886464,
"pooled_auc_strict": 0.9391905508894626,
"spec_top1": 0.7083333333333334,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.6999603235345861,
"cat_p5": 0.775,
"cat_p10": 0.6375000000000001,
"class_map": 0.6421799826780327,
"class_top3": 0.8225806451612904,
"tag_acc": 0.5806451612903226,
"tag_mrr": 0.711020057978075,
"pooled_auc": 0.9625156962704589,
"pooled_auc_strict": 0.9483321470937129,
"spec_top1": 0.6,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.6923689595029523,
"cat_p5": 0.7,
"cat_p10": 0.6375,
"class_map": 0.6342054217314276,
"class_top3": 0.7931034482758621,
"tag_acc": 0.5526315789473685,
"tag_mrr": 0.6857341096884034,
"pooled_auc": 0.9431299790133811,
"pooled_auc_strict": 0.9273655769746675,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "zero",
"templates": "t1",
"method": "both",
"all": {
"cat_map": 0.7425693181695112,
"cat_p5": 0.85,
"cat_p10": 0.8125,
"class_map": 0.60919922426989,
"class_top3": 0.8571428571428571,
"tag_acc": 0.5684931506849316,
"tag_mrr": 0.700051267624108,
"pooled_auc": 0.9584685023879301,
"pooled_auc_strict": 0.9439301756411793,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.743093620970253,
"cat_p5": 0.825,
"cat_p10": 0.6875,
"class_map": 0.6592606009695009,
"class_top3": 0.8387096774193549,
"tag_acc": 0.5806451612903226,
"tag_mrr": 0.711020057978075,
"pooled_auc": 0.9637552991682318,
"pooled_auc_strict": 0.9497508896797153,
"spec_top1": 0.8,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.747603229487134,
"cat_p5": 0.775,
"cat_p10": 0.7125,
"class_map": 0.6141318587911977,
"class_top3": 0.7931034482758621,
"tag_acc": 0.5526315789473685,
"tag_mrr": 0.6857341096884034,
"pooled_auc": 0.9515237417761121,
"pooled_auc_strict": 0.9362305024336028,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "zero",
"templates": "t1",
"method": "term_sd",
"all": {
"cat_map": 0.6966951514439714,
"cat_p5": 0.9,
"cat_p10": 0.75,
"class_map": 0.6103205175324865,
"class_top3": 0.8428571428571429,
"tag_acc": 0.5662100456621004,
"tag_mrr": 0.6955952165165974,
"pooled_auc": 0.9520811285683274,
"pooled_auc_strict": 0.9379881695480244,
"spec_top1": 0.7083333333333334,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.6999603235345861,
"cat_p5": 0.775,
"cat_p10": 0.6375000000000001,
"class_map": 0.6421799826780327,
"class_top3": 0.8225806451612904,
"tag_acc": 0.5766129032258065,
"tag_mrr": 0.7039788209421436,
"pooled_auc": 0.9599737053930776,
"pooled_auc_strict": 0.9465689205219454,
"spec_top1": 0.6,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.6923689595029523,
"cat_p5": 0.7,
"cat_p10": 0.6375,
"class_map": 0.6342054217314276,
"class_top3": 0.7931034482758621,
"tag_acc": 0.5526315789473685,
"tag_mrr": 0.6846524065295687,
"pooled_auc": 0.9417612480137032,
"pooled_auc_strict": 0.9267723463508731,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "zero",
"templates": "t1",
"method": "both_sd",
"all": {
"cat_map": 0.7357901241335416,
"cat_p5": 0.825,
"cat_p10": 0.8,
"class_map": 0.5762451701876967,
"class_top3": 0.8142857142857143,
"tag_acc": 0.5662100456621004,
"tag_mrr": 0.6955952165165974,
"pooled_auc": 0.9544702458482579,
"pooled_auc_strict": 0.9397197983957355,
"spec_top1": 0.875,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7328908579222572,
"cat_p5": 0.8,
"cat_p10": 0.675,
"class_map": 0.6431110174811173,
"class_top3": 0.8225806451612904,
"tag_acc": 0.5766129032258065,
"tag_mrr": 0.7039788209421436,
"pooled_auc": 0.9587121008854307,
"pooled_auc_strict": 0.9446315539739027,
"spec_top1": 0.8,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7635682012315999,
"cat_p5": 0.775,
"cat_p10": 0.6749999999999999,
"class_map": 0.6058761528294955,
"class_top3": 0.8275862068965517,
"tag_acc": 0.5526315789473685,
"tag_mrr": 0.6846524065295687,
"pooled_auc": 0.9489826975414158,
"pooled_auc_strict": 0.9332092579108062,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "zero",
"templates": "t3",
"method": "raw",
"all": {
"cat_map": 0.7113266964725198,
"cat_p5": 0.85,
"cat_p10": 0.7749999999999999,
"class_map": 0.6242070802503793,
"class_top3": 0.8428571428571429,
"tag_acc": 0.589041095890411,
"tag_mrr": 0.7051300820155875,
"pooled_auc": 0.9528507611527189,
"pooled_auc_strict": 0.9377459381993843,
"spec_top1": 0.75,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7082306928343325,
"cat_p5": 0.825,
"cat_p10": 0.7124999999999999,
"class_map": 0.6412667770398142,
"class_top3": 0.8548387096774194,
"tag_acc": 0.6209677419354839,
"tag_mrr": 0.7232177412169984,
"pooled_auc": 0.96023557821304,
"pooled_auc_strict": 0.9453508896797153,
"spec_top1": 0.6,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7129527355473899,
"cat_p5": 0.75,
"cat_p10": 0.6625,
"class_map": 0.6568296279529588,
"class_top3": 0.7931034482758621,
"tag_acc": 0.5473684210526316,
"tag_mrr": 0.68152092684743,
"pooled_auc": 0.9435396250721083,
"pooled_auc_strict": 0.9281328644975025,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "zero",
"templates": "t3",
"method": "sound",
"all": {
"cat_map": 0.7479251873265214,
"cat_p5": 0.85,
"cat_p10": 0.8374999999999999,
"class_map": 0.6260961543743475,
"class_top3": 0.8571428571428571,
"tag_acc": 0.589041095890411,
"tag_mrr": 0.7051300820155875,
"pooled_auc": 0.9592016715510692,
"pooled_auc_strict": 0.9445763780709368,
"spec_top1": 0.875,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7428199736405021,
"cat_p5": 0.875,
"cat_p10": 0.7,
"class_map": 0.6625634652022035,
"class_top3": 0.8870967741935484,
"tag_acc": 0.6209677419354839,
"tag_mrr": 0.7232177412169984,
"pooled_auc": 0.9641454252750201,
"pooled_auc_strict": 0.9495302491103202,
"spec_top1": 0.8,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7732243482406984,
"cat_p5": 0.8,
"cat_p10": 0.7000000000000001,
"class_map": 0.619360899639458,
"class_top3": 0.8103448275862069,
"tag_acc": 0.5473684210526316,
"tag_mrr": 0.68152092684743,
"pooled_auc": 0.9527509104228822,
"pooled_auc_strict": 0.9381084007204997,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "zero",
"templates": "t3",
"method": "center",
"all": {
"cat_map": 0.7113266964725198,
"cat_p5": 0.85,
"cat_p10": 0.7749999999999999,
"class_map": 0.6242070802503793,
"class_top3": 0.8428571428571429,
"tag_acc": 0.5730593607305936,
"tag_mrr": 0.7024972971901691,
"pooled_auc": 0.9569955497666341,
"pooled_auc_strict": 0.9427986027139594,
"spec_top1": 0.75,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7082306928343325,
"cat_p5": 0.825,
"cat_p10": 0.7124999999999999,
"class_map": 0.6412667770398142,
"class_top3": 0.8548387096774194,
"tag_acc": 0.6008064516129032,
"tag_mrr": 0.7192411307330662,
"pooled_auc": 0.9652535551381809,
"pooled_auc_strict": 0.9514296559905101,
"spec_top1": 0.6,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7129527355473899,
"cat_p5": 0.75,
"cat_p10": 0.6625,
"class_map": 0.6568296279529588,
"class_top3": 0.7931034482758621,
"tag_acc": 0.5368421052631579,
"tag_mrr": 0.6806421881447032,
"pooled_auc": 0.9465230516596417,
"pooled_auc_strict": 0.9319054280202864,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "zero",
"templates": "t3",
"method": "both",
"all": {
"cat_map": 0.7545544072897863,
"cat_p5": 0.85,
"cat_p10": 0.8374999999999999,
"class_map": 0.6295778568154784,
"class_top3": 0.8428571428571429,
"tag_acc": 0.5730593607305936,
"tag_mrr": 0.7024972971901691,
"pooled_auc": 0.9623862002062303,
"pooled_auc_strict": 0.9479070969397451,
"spec_top1": 0.9166666666666666,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7569497491591481,
"cat_p5": 0.85,
"cat_p10": 0.7374999999999999,
"class_map": 0.6593895422627897,
"class_top3": 0.8387096774193549,
"tag_acc": 0.6008064516129032,
"tag_mrr": 0.7192411307330662,
"pooled_auc": 0.9682763616850013,
"pooled_auc_strict": 0.9543800711743772,
"spec_top1": 0.85,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7768694680931955,
"cat_p5": 0.8,
"cat_p10": 0.725,
"class_map": 0.6320349511109097,
"class_top3": 0.8275862068965517,
"tag_acc": 0.5368421052631579,
"tag_mrr": 0.6806421881447032,
"pooled_auc": 0.9545637933048086,
"pooled_auc_strict": 0.9393882778267479,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "zero",
"templates": "t3",
"method": "term_sd",
"all": {
"cat_map": 0.7113266964725198,
"cat_p5": 0.85,
"cat_p10": 0.7749999999999999,
"class_map": 0.6242070802503793,
"class_top3": 0.8428571428571429,
"tag_acc": 0.5707762557077626,
"tag_mrr": 0.7004832234156526,
"pooled_auc": 0.9547480462390101,
"pooled_auc_strict": 0.9412084393946546,
"spec_top1": 0.75,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7082306928343325,
"cat_p5": 0.825,
"cat_p10": 0.7124999999999999,
"class_map": 0.6412667770398142,
"class_top3": 0.8548387096774194,
"tag_acc": 0.5806451612903226,
"tag_mrr": 0.7099978762277149,
"pooled_auc": 0.9625784813522941,
"pooled_auc_strict": 0.9494111506524318,
"spec_top1": 0.6,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7129527355473899,
"cat_p5": 0.75,
"cat_p10": 0.6625,
"class_map": 0.6568296279529588,
"class_top3": 0.7931034482758621,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6880640976399084,
"pooled_auc": 0.9446561981306691,
"pooled_auc_strict": 0.9306151713741873,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "zero",
"templates": "t3",
"method": "both_sd",
"all": {
"cat_map": 0.7419979603370317,
"cat_p5": 0.825,
"cat_p10": 0.8125,
"class_map": 0.6137491601963941,
"class_top3": 0.8142857142857143,
"tag_acc": 0.5707762557077626,
"tag_mrr": 0.7004832234156526,
"pooled_auc": 0.9591180600781505,
"pooled_auc_strict": 0.9443836218322312,
"spec_top1": 0.9166666666666666,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7383885442934307,
"cat_p5": 0.8,
"cat_p10": 0.7,
"class_map": 0.6636712966205031,
"class_top3": 0.8387096774193549,
"tag_acc": 0.5806451612903226,
"tag_mrr": 0.7099978762277149,
"pooled_auc": 0.9639930238797961,
"pooled_auc_strict": 0.9498642941874259,
"spec_top1": 0.8,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7736227636847813,
"cat_p5": 0.8,
"cat_p10": 0.6875,
"class_map": 0.6297619157865565,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6880640976399084,
"pooled_auc": 0.9527447170699425,
"pooled_auc_strict": 0.9371798311169024,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "zero",
"templates": "t5",
"method": "raw",
"all": {
"cat_map": 0.7195432854073197,
"cat_p5": 0.875,
"cat_p10": 0.7875,
"class_map": 0.6207924807074807,
"class_top3": 0.8285714285714286,
"tag_acc": 0.6004566210045662,
"tag_mrr": 0.7122834126130668,
"pooled_auc": 0.9552127428633452,
"pooled_auc_strict": 0.9398409897200707,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7192035638255845,
"cat_p5": 0.825,
"cat_p10": 0.7124999999999999,
"class_map": 0.6365957342756754,
"class_top3": 0.8548387096774194,
"tag_acc": 0.6209677419354839,
"tag_mrr": 0.7251502314078777,
"pooled_auc": 0.96239602897773,
"pooled_auc_strict": 0.9472868327402135,
"spec_top1": 0.7,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7103068995186603,
"cat_p5": 0.75,
"cat_p10": 0.6375,
"class_map": 0.6498509450129192,
"class_top3": 0.7931034482758621,
"tag_acc": 0.5736842105263158,
"tag_mrr": 0.6954888280808925,
"pooled_auc": 0.9461948039538365,
"pooled_auc_strict": 0.9304483003104281,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "zero",
"templates": "t5",
"method": "sound",
"all": {
"cat_map": 0.7528268584243417,
"cat_p5": 0.85,
"cat_p10": 0.8500000000000001,
"class_map": 0.631513081220779,
"class_top3": 0.8714285714285714,
"tag_acc": 0.6004566210045662,
"tag_mrr": 0.7122834126130668,
"pooled_auc": 0.9614140684359058,
"pooled_auc_strict": 0.946796857316809,
"spec_top1": 0.875,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7561693052788976,
"cat_p5": 0.8999999999999999,
"cat_p10": 0.725,
"class_map": 0.6608365270149242,
"class_top3": 0.8870967741935484,
"tag_acc": 0.6209677419354839,
"tag_mrr": 0.7251502314078777,
"pooled_auc": 0.9661851354977193,
"pooled_auc_strict": 0.9514424673784104,
"spec_top1": 0.8,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7558766882096197,
"cat_p5": 0.8,
"cat_p10": 0.6875,
"class_map": 0.6321180749414527,
"class_top3": 0.8275862068965517,
"tag_acc": 0.5736842105263158,
"tag_mrr": 0.6954888280808925,
"pooled_auc": 0.9552176344223018,
"pooled_auc_strict": 0.9407823298713576,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "zero",
"templates": "t5",
"method": "center",
"all": {
"cat_map": 0.7195432854073197,
"cat_p5": 0.875,
"cat_p10": 0.7875,
"class_map": 0.6207924807074807,
"class_top3": 0.8285714285714286,
"tag_acc": 0.591324200913242,
"tag_mrr": 0.713834516703891,
"pooled_auc": 0.9582488738738739,
"pooled_auc_strict": 0.9438378826226468,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7192035638255845,
"cat_p5": 0.825,
"cat_p10": 0.7124999999999999,
"class_map": 0.6365957342756754,
"class_top3": 0.8548387096774194,
"tag_acc": 0.5967741935483871,
"tag_mrr": 0.7193428063719185,
"pooled_auc": 0.9658282801180574,
"pooled_auc_strict": 0.951911743772242,
"spec_top1": 0.7,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7103068995186603,
"cat_p5": 0.75,
"cat_p10": 0.6375,
"class_map": 0.6498509450129192,
"class_top3": 0.7931034482758621,
"tag_acc": 0.5842105263157895,
"tag_mrr": 0.7066447491372023,
"pooled_auc": 0.9485571257179866,
"pooled_auc_strict": 0.9336044788512884,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "zero",
"templates": "t5",
"method": "both",
"all": {
"cat_map": 0.7583908419074668,
"cat_p5": 0.85,
"cat_p10": 0.8374999999999999,
"class_map": 0.631183476723012,
"class_top3": 0.8428571428571429,
"tag_acc": 0.591324200913242,
"tag_mrr": 0.713834516703891,
"pooled_auc": 0.9637078720286552,
"pooled_auc_strict": 0.9492772700148758,
"spec_top1": 0.875,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7592935004031254,
"cat_p5": 0.85,
"cat_p10": 0.7374999999999999,
"class_map": 0.6625594786345831,
"class_top3": 0.8548387096774194,
"tag_acc": 0.5967741935483871,
"tag_mrr": 0.7193428063719185,
"pooled_auc": 0.9691033002414811,
"pooled_auc_strict": 0.9551772241992883,
"spec_top1": 0.85,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7741438076784091,
"cat_p5": 0.8,
"cat_p10": 0.725,
"class_map": 0.6288217086231476,
"class_top3": 0.8793103448275862,
"tag_acc": 0.5842105263157895,
"tag_mrr": 0.7066447491372023,
"pooled_auc": 0.9565430119514017,
"pooled_auc_strict": 0.9414977196949373,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "zero",
"templates": "t5",
"method": "term_sd",
"all": {
"cat_map": 0.7195432854073197,
"cat_p5": 0.875,
"cat_p10": 0.7875,
"class_map": 0.6207924807074807,
"class_top3": 0.8285714285714286,
"tag_acc": 0.5753424657534246,
"tag_mrr": 0.7039218843087036,
"pooled_auc": 0.9559677900792358,
"pooled_auc_strict": 0.9423465182231808,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7192035638255845,
"cat_p5": 0.825,
"cat_p10": 0.7124999999999999,
"class_map": 0.6365957342756754,
"class_top3": 0.8548387096774194,
"tag_acc": 0.5887096774193549,
"tag_mrr": 0.7139980675215183,
"pooled_auc": 0.9630893480010733,
"pooled_auc_strict": 0.9499814946619217,
"spec_top1": 0.7,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7103068995186603,
"cat_p5": 0.75,
"cat_p10": 0.6375,
"class_map": 0.6498509450129192,
"class_top3": 0.7931034482758621,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6907698135888196,
"pooled_auc": 0.9468663403654433,
"pooled_auc_strict": 0.9326407784974259,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.8571428571428571
}
},
{
"pad": "zero",
"templates": "t5",
"method": "both_sd",
"all": {
"cat_map": 0.7440666122282521,
"cat_p5": 0.85,
"cat_p10": 0.8,
"class_map": 0.6171414447933016,
"class_top3": 0.8428571428571429,
"tag_acc": 0.5753424657534246,
"tag_mrr": 0.7039218843087036,
"pooled_auc": 0.9602011084880061,
"pooled_auc_strict": 0.9455557431675933,
"spec_top1": 0.875,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7428221662902166,
"cat_p5": 0.8,
"cat_p10": 0.7124999999999999,
"class_map": 0.6513844220634469,
"class_top3": 0.8548387096774194,
"tag_acc": 0.5887096774193549,
"tag_mrr": 0.7139980675215183,
"pooled_auc": 0.9645468204990609,
"pooled_auc_strict": 0.9504583629893238,
"spec_top1": 0.8,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7663582183951299,
"cat_p5": 0.8,
"cat_p10": 0.7000000000000001,
"class_map": 0.6256447411677083,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6907698135888196,
"pooled_auc": 0.9544602758342446,
"pooled_auc_strict": 0.9390521404208025,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t1",
"method": "raw",
"all": {
"cat_map": 0.706922872057108,
"cat_p5": 0.875,
"cat_p10": 0.7875000000000001,
"class_map": 0.6107295823937894,
"class_top3": 0.8714285714285714,
"tag_acc": 0.5616438356164384,
"tag_mrr": 0.6909010612328351,
"pooled_auc": 0.9446412338543363,
"pooled_auc_strict": 0.9287270946884006,
"spec_top1": 0.75,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.6884970537189887,
"cat_p5": 0.75,
"cat_p10": 0.6499999999999999,
"class_map": 0.6423744153944477,
"class_top3": 0.8548387096774194,
"tag_acc": 0.592741935483871,
"tag_mrr": 0.7138462062406248,
"pooled_auc": 0.9530222699221894,
"pooled_auc_strict": 0.9374448398576513,
"spec_top1": 0.7,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7079575497135653,
"cat_p5": 0.75,
"cat_p10": 0.6625000000000001,
"class_map": 0.6028376226217312,
"class_top3": 0.7758620689655172,
"tag_acc": 0.5210526315789473,
"tag_mrr": 0.660951608801615,
"pooled_auc": 0.93430268154487,
"pooled_auc_strict": 0.917817198737848,
"spec_top1": 0.6190476190476191,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t1",
"method": "sound",
"all": {
"cat_map": 0.7490033176025689,
"cat_p5": 0.8500000000000001,
"cat_p10": 0.825,
"class_map": 0.6199766764100778,
"class_top3": 0.8714285714285714,
"tag_acc": 0.5616438356164384,
"tag_mrr": 0.6909010612328351,
"pooled_auc": 0.956998772115489,
"pooled_auc_strict": 0.940935040239756,
"spec_top1": 0.875,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7477036476412733,
"cat_p5": 0.825,
"cat_p10": 0.7375,
"class_map": 0.6691017688292265,
"class_top3": 0.8870967741935484,
"tag_acc": 0.592741935483871,
"tag_mrr": 0.7138462062406248,
"pooled_auc": 0.9636592433592702,
"pooled_auc_strict": 0.9478372479240806,
"spec_top1": 0.9,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7696312428996632,
"cat_p5": 0.8,
"cat_p10": 0.7,
"class_map": 0.6155754649787434,
"class_top3": 0.7931034482758621,
"tag_acc": 0.5210526315789473,
"tag_mrr": 0.660951608801615,
"pooled_auc": 0.948381942306263,
"pooled_auc_strict": 0.9320555321350554,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t1",
"method": "center",
"all": {
"cat_map": 0.706922872057108,
"cat_p5": 0.875,
"cat_p10": 0.7875000000000001,
"class_map": 0.6107295823937894,
"class_top3": 0.8714285714285714,
"tag_acc": 0.5776255707762558,
"tag_mrr": 0.7087323497675215,
"pooled_auc": 0.949996608053837,
"pooled_auc_strict": 0.9345255170527238,
"spec_top1": 0.75,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.6884970537189887,
"cat_p5": 0.75,
"cat_p10": 0.6499999999999999,
"class_map": 0.6423744153944477,
"class_top3": 0.8548387096774194,
"tag_acc": 0.6129032258064516,
"tag_mrr": 0.7366968087148714,
"pooled_auc": 0.9584314462033807,
"pooled_auc_strict": 0.9437812574139977,
"spec_top1": 0.7,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7079575497135653,
"cat_p5": 0.75,
"cat_p10": 0.6625000000000001,
"class_map": 0.6028376226217312,
"class_top3": 0.7758620689655172,
"tag_acc": 0.531578947368421,
"tag_mrr": 0.6722313717730859,
"pooled_auc": 0.9392547096025283,
"pooled_auc_strict": 0.922591787069329,
"spec_top1": 0.6190476190476191,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t1",
"method": "both",
"all": {
"cat_map": 0.7673928880987917,
"cat_p5": 0.85,
"cat_p10": 0.825,
"class_map": 0.6187623766919265,
"class_top3": 0.8571428571428571,
"tag_acc": 0.5776255707762558,
"tag_mrr": 0.7087323497675215,
"pooled_auc": 0.9614535846087051,
"pooled_auc_strict": 0.9457649911096102,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7531848837739687,
"cat_p5": 0.8,
"cat_p10": 0.7374999999999999,
"class_map": 0.6936505103141615,
"class_top3": 0.9032258064516129,
"tag_acc": 0.6129032258064516,
"tag_mrr": 0.7366968087148714,
"pooled_auc": 0.9673871746713174,
"pooled_auc_strict": 0.9529214709371293,
"spec_top1": 0.85,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7791705700145516,
"cat_p5": 0.8,
"cat_p10": 0.7250000000000001,
"class_map": 0.6199613689407614,
"class_top3": 0.8275862068965517,
"tag_acc": 0.531578947368421,
"tag_mrr": 0.6722313717730859,
"pooled_auc": 0.9536958391285422,
"pooled_auc_strict": 0.9363742191392328,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t1",
"method": "term_sd",
"all": {
"cat_map": 0.706922872057108,
"cat_p5": 0.875,
"cat_p10": 0.7875000000000001,
"class_map": 0.6107295823937894,
"class_top3": 0.8714285714285714,
"tag_acc": 0.54337899543379,
"tag_mrr": 0.6884737470690645,
"pooled_auc": 0.947702126071855,
"pooled_auc_strict": 0.9326695195800395,
"spec_top1": 0.75,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.6884970537189887,
"cat_p5": 0.75,
"cat_p10": 0.6499999999999999,
"class_map": 0.6423744153944477,
"class_top3": 0.8548387096774194,
"tag_acc": 0.592741935483871,
"tag_mrr": 0.7203287804617625,
"pooled_auc": 0.9560660048296217,
"pooled_auc_strict": 0.941944009489917,
"spec_top1": 0.7,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7079575497135653,
"cat_p5": 0.75,
"cat_p10": 0.6625000000000001,
"class_map": 0.6028376226217312,
"class_top3": 0.7758620689655172,
"tag_acc": 0.4789473684210526,
"tag_mrr": 0.6468945455880694,
"pooled_auc": 0.9369729014265946,
"pooled_auc_strict": 0.920643627281902,
"spec_top1": 0.6190476190476191,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t1",
"method": "both_sd",
"all": {
"cat_map": 0.7663527622016136,
"cat_p5": 0.85,
"cat_p10": 0.8125,
"class_map": 0.6053154405539275,
"class_top3": 0.8428571428571429,
"tag_acc": 0.54337899543379,
"tag_mrr": 0.6884737470690645,
"pooled_auc": 0.9586613345272984,
"pooled_auc_strict": 0.9423814685301989,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9166666666666666
},
"dev": {
"cat_map": 0.7489604716512064,
"cat_p5": 0.8,
"cat_p10": 0.75,
"class_map": 0.6948401730457527,
"class_top3": 0.9032258064516129,
"tag_acc": 0.592741935483871,
"tag_mrr": 0.7203287804617625,
"pooled_auc": 0.9639989267507378,
"pooled_auc_strict": 0.9490481613285884,
"spec_top1": 0.85,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7970968265684556,
"cat_p5": 0.8,
"cat_p10": 0.725,
"class_map": 0.6219079826197252,
"class_top3": 0.8275862068965517,
"tag_acc": 0.4789473684210526,
"tag_mrr": 0.6468945455880694,
"pooled_auc": 0.9516599955407858,
"pooled_auc_strict": 0.9335222409586224,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t3",
"method": "raw",
"all": {
"cat_map": 0.733627480353698,
"cat_p5": 0.875,
"cat_p10": 0.8125,
"class_map": 0.631684550946142,
"class_top3": 0.8857142857142857,
"tag_acc": 0.591324200913242,
"tag_mrr": 0.7090230796224233,
"pooled_auc": 0.9471253256268316,
"pooled_auc_strict": 0.9317075539868769,
"spec_top1": 0.875,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.721332154539629,
"cat_p5": 0.825,
"cat_p10": 0.7,
"class_map": 0.6685541340636711,
"class_top3": 0.8548387096774194,
"tag_acc": 0.625,
"tag_mrr": 0.7321579254982588,
"pooled_auc": 0.955209551918433,
"pooled_auc_strict": 0.9403696322657177,
"spec_top1": 0.8,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7408307618711064,
"cat_p5": 0.775,
"cat_p10": 0.675,
"class_map": 0.6341914042688647,
"class_top3": 0.8275862068965517,
"tag_acc": 0.5473684210526316,
"tag_mrr": 0.6788260176371224,
"pooled_auc": 0.9368516886619173,
"pooled_auc_strict": 0.9206635879354617,
"spec_top1": 0.7142857142857143,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t3",
"method": "sound",
"all": {
"cat_map": 0.7712992046064168,
"cat_p5": 0.85,
"cat_p10": 0.8375,
"class_map": 0.6432533280537299,
"class_top3": 0.9,
"tag_acc": 0.591324200913242,
"tag_mrr": 0.7090230796224233,
"pooled_auc": 0.959616845761424,
"pooled_auc_strict": 0.9441547048862647,
"spec_top1": 0.8333333333333334,
"spec_top3": 1.0
},
"dev": {
"cat_map": 0.7656542451859354,
"cat_p5": 0.875,
"cat_p10": 0.7375,
"class_map": 0.6948406693679899,
"class_top3": 0.9193548387096774,
"tag_acc": 0.625,
"tag_mrr": 0.7321579254982588,
"pooled_auc": 0.9671735980681513,
"pooled_auc_strict": 0.9523753262158956,
"spec_top1": 0.85,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7943363173833515,
"cat_p5": 0.775,
"cat_p10": 0.7375,
"class_map": 0.6240989688025537,
"class_top3": 0.8275862068965517,
"tag_acc": 0.5473684210526316,
"tag_mrr": 0.6788260176371224,
"pooled_auc": 0.9499099309529624,
"pooled_auc_strict": 0.9336116646865699,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t3",
"method": "center",
"all": {
"cat_map": 0.733627480353698,
"cat_p5": 0.875,
"cat_p10": 0.8125,
"class_map": 0.631684550946142,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5821917808219178,
"tag_mrr": 0.7082388382353201,
"pooled_auc": 0.9511218861934223,
"pooled_auc_strict": 0.93656489016223,
"spec_top1": 0.875,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.721332154539629,
"cat_p5": 0.825,
"cat_p10": 0.7,
"class_map": 0.6685541340636711,
"class_top3": 0.8548387096774194,
"tag_acc": 0.6129032258064516,
"tag_mrr": 0.7332454862192767,
"pooled_auc": 0.9593807351757445,
"pooled_auc_strict": 0.9456540925266904,
"spec_top1": 0.8,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7408307618711064,
"cat_p5": 0.775,
"cat_p10": 0.675,
"class_map": 0.6341914042688647,
"class_top3": 0.8275862068965517,
"tag_acc": 0.5421052631578948,
"tag_mrr": 0.6755985819194191,
"pooled_auc": 0.9407968544845184,
"pooled_auc_strict": 0.9251156121054178,
"spec_top1": 0.7142857142857143,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t3",
"method": "both",
"all": {
"cat_map": 0.775188178344212,
"cat_p5": 0.85,
"cat_p10": 0.825,
"class_map": 0.6431672182428108,
"class_top3": 0.9,
"tag_acc": 0.5821917808219178,
"tag_mrr": 0.7082388382353201,
"pooled_auc": 0.9633947953978075,
"pooled_auc_strict": 0.9482859522158192,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7703344479824905,
"cat_p5": 0.85,
"cat_p10": 0.75,
"class_map": 0.6863326451282192,
"class_top3": 0.9193548387096774,
"tag_acc": 0.6129032258064516,
"tag_mrr": 0.7332454862192767,
"pooled_auc": 0.9702516769519721,
"pooled_auc_strict": 0.9564493475682088,
"spec_top1": 0.9,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7965686178706268,
"cat_p5": 0.775,
"cat_p10": 0.7375,
"class_map": 0.6331310086379538,
"class_top3": 0.8275862068965517,
"tag_acc": 0.5421052631578948,
"tag_mrr": 0.6755985819194191,
"pooled_auc": 0.9545230941283475,
"pooled_auc_strict": 0.9377227608937263,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t3",
"method": "term_sd",
"all": {
"cat_map": 0.733627480353698,
"cat_p5": 0.875,
"cat_p10": 0.8125,
"class_map": 0.631684550946142,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5639269406392694,
"tag_mrr": 0.699348237848909,
"pooled_auc": 0.9491552358080972,
"pooled_auc_strict": 0.9350142161508546,
"spec_top1": 0.875,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.721332154539629,
"cat_p5": 0.825,
"cat_p10": 0.7,
"class_map": 0.6685541340636711,
"class_top3": 0.8548387096774194,
"tag_acc": 0.5967741935483871,
"tag_mrr": 0.7250881387877356,
"pooled_auc": 0.9572465790179769,
"pooled_auc_strict": 0.9439345195729537,
"spec_top1": 0.8,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7408307618711064,
"cat_p5": 0.775,
"cat_p10": 0.675,
"class_map": 0.6341914042688647,
"class_top3": 0.8275862068965517,
"tag_acc": 0.5210526315789473,
"tag_mrr": 0.6657508934655983,
"pooled_auc": 0.9390768718966878,
"pooled_auc_strict": 0.9237942168397655,
"spec_top1": 0.7142857142857143,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t3",
"method": "both_sd",
"all": {
"cat_map": 0.7682368356425465,
"cat_p5": 0.85,
"cat_p10": 0.8125,
"class_map": 0.6256953056101257,
"class_top3": 0.8714285714285714,
"tag_acc": 0.5639269406392694,
"tag_mrr": 0.699348237848909,
"pooled_auc": 0.961727653858678,
"pooled_auc_strict": 0.9461088960786663,
"spec_top1": 0.875,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7569966515589786,
"cat_p5": 0.825,
"cat_p10": 0.7375,
"class_map": 0.6843928205030713,
"class_top3": 0.9193548387096774,
"tag_acc": 0.5967741935483871,
"tag_mrr": 0.7250881387877356,
"pooled_auc": 0.9678078883820768,
"pooled_auc_strict": 0.9535316725978648,
"spec_top1": 0.9,
"spec_top3": 0.95
},
"test": {
"cat_map": 0.7923047240435662,
"cat_p5": 0.8,
"cat_p10": 0.725,
"class_map": 0.6436344180966361,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5210526315789473,
"tag_mrr": 0.6657508934655983,
"pooled_auc": 0.9538444795990955,
"pooled_auc_strict": 0.9364492711966173,
"spec_top1": 0.9047619047619048,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t5",
"method": "raw",
"all": {
"cat_map": 0.7403152944695368,
"cat_p5": 0.875,
"cat_p10": 0.8375,
"class_map": 0.6306648972281436,
"class_top3": 0.8857142857142857,
"tag_acc": 0.6004566210045662,
"tag_mrr": 0.7167575604039297,
"pooled_auc": 0.9505566183653533,
"pooled_auc_strict": 0.935182764384699,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7288654463711456,
"cat_p5": 0.85,
"cat_p10": 0.7,
"class_map": 0.6623461312956833,
"class_top3": 0.8387096774193549,
"tag_acc": 0.625,
"tag_mrr": 0.7350574492825074,
"pooled_auc": 0.9579908773812718,
"pooled_auc_strict": 0.9432763938315539,
"spec_top1": 0.8,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7339504300278596,
"cat_p5": 0.775,
"cat_p10": 0.675,
"class_map": 0.644370763116193,
"class_top3": 0.8103448275862069,
"tag_acc": 0.5684210526315789,
"tag_mrr": 0.6928713896571546,
"pooled_auc": 0.9411640318373732,
"pooled_auc_strict": 0.9250150104114769,
"spec_top1": 0.7142857142857143,
"spec_top3": 0.9523809523809523
}
},
{
"pad": "spaced",
"templates": "t5",
"method": "sound",
"all": {
"cat_map": 0.7754300541392372,
"cat_p5": 0.85,
"cat_p10": 0.8625,
"class_map": 0.6522621153856848,
"class_top3": 0.9142857142857143,
"tag_acc": 0.6004566210045662,
"tag_mrr": 0.7167575604039297,
"pooled_auc": 0.9627605014653208,
"pooled_auc_strict": 0.947461972250364,
"spec_top1": 0.8333333333333334,
"spec_top3": 1.0
},
"dev": {
"cat_map": 0.7717425709437424,
"cat_p5": 0.8999999999999999,
"cat_p10": 0.75,
"class_map": 0.6858513149747846,
"class_top3": 0.9193548387096774,
"tag_acc": 0.625,
"tag_mrr": 0.7350574492825074,
"pooled_auc": 0.969741346927824,
"pooled_auc_strict": 0.9549988137603797,
"spec_top1": 0.85,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7797688455311863,
"cat_p5": 0.8,
"cat_p10": 0.7375,
"class_map": 0.6508153250257953,
"class_top3": 0.8620689655172413,
"tag_acc": 0.5684210526315789,
"tag_mrr": 0.6928713896571546,
"pooled_auc": 0.9538878330696734,
"pooled_auc_strict": 0.9379255611338929,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t5",
"method": "center",
"all": {
"cat_map": 0.7403152944695368,
"cat_p5": 0.875,
"cat_p10": 0.8375,
"class_map": 0.6306648972281436,
"class_top3": 0.8857142857142857,
"tag_acc": 0.589041095890411,
"tag_mrr": 0.7166557925025382,
"pooled_auc": 0.9532756024096386,
"pooled_auc_strict": 0.9387681212046387,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7288654463711456,
"cat_p5": 0.85,
"cat_p10": 0.7,
"class_map": 0.6623461312956833,
"class_top3": 0.8387096774193549,
"tag_acc": 0.6048387096774194,
"tag_mrr": 0.7341978621034598,
"pooled_auc": 0.9607158572578481,
"pooled_auc_strict": 0.9471805456702254,
"spec_top1": 0.8,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7339504300278596,
"cat_p5": 0.775,
"cat_p10": 0.675,
"class_map": 0.644370763116193,
"class_top3": 0.8103448275862069,
"tag_acc": 0.5684210526315789,
"tag_mrr": 0.69375877533923,
"pooled_auc": 0.9439333453661334,
"pooled_auc_strict": 0.9281464377419231,
"spec_top1": 0.7142857142857143,
"spec_top3": 0.9523809523809523
}
},
{
"pad": "spaced",
"templates": "t5",
"method": "both",
"all": {
"cat_map": 0.7842722653956207,
"cat_p5": 0.85,
"cat_p10": 0.8500000000000001,
"class_map": 0.6437703791946205,
"class_top3": 0.8714285714285714,
"tag_acc": 0.589041095890411,
"tag_mrr": 0.7166557925025382,
"pooled_auc": 0.965359749810051,
"pooled_auc_strict": 0.9504587719521214,
"spec_top1": 0.7916666666666666,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7786263627759495,
"cat_p5": 0.85,
"cat_p10": 0.75,
"class_map": 0.6880803109400951,
"class_top3": 0.9193548387096774,
"tag_acc": 0.6048387096774194,
"tag_mrr": 0.7341978621034598,
"pooled_auc": 0.9716114837671049,
"pooled_auc_strict": 0.9579487544483986,
"spec_top1": 0.9,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.8002964532472743,
"cat_p5": 0.8,
"cat_p10": 0.7375,
"class_map": 0.6358470829434432,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5684210526315789,
"tag_mrr": 0.69375877533923,
"pooled_auc": 0.9572817904806395,
"pooled_auc_strict": 0.9407392148596686,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
}
},
{
"pad": "spaced",
"templates": "t5",
"method": "term_sd",
"all": {
"cat_map": 0.7403152944695368,
"cat_p5": 0.875,
"cat_p10": 0.8375,
"class_map": 0.6306648972281436,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5799086757990868,
"tag_mrr": 0.7094754522618477,
"pooled_auc": 0.9510448890155215,
"pooled_auc_strict": 0.9370711400638849,
"spec_top1": 0.8333333333333334,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7288654463711456,
"cat_p5": 0.85,
"cat_p10": 0.7,
"class_map": 0.6623461312956833,
"class_top3": 0.8387096774193549,
"tag_acc": 0.5967741935483871,
"tag_mrr": 0.7250974680232269,
"pooled_auc": 0.9583783203649048,
"pooled_auc_strict": 0.9453580071174377,
"spec_top1": 0.8,
"spec_top3": 0.9
},
"test": {
"cat_map": 0.7339504300278596,
"cat_p5": 0.775,
"cat_p10": 0.675,
"class_map": 0.644370763116193,
"class_top3": 0.8103448275862069,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6890846106364683,
"pooled_auc": 0.9418727283666182,
"pooled_auc_strict": 0.9265998863041173,
"spec_top1": 0.7142857142857143,
"spec_top3": 0.9523809523809523
}
},
{
"pad": "spaced",
"templates": "t5",
"method": "both_sd",
"all": {
"cat_map": 0.773238958326166,
"cat_p5": 0.85,
"cat_p10": 0.8,
"class_map": 0.634303067677147,
"class_top3": 0.8857142857142857,
"tag_acc": 0.5799086757990868,
"tag_mrr": 0.7094754522618477,
"pooled_auc": 0.9635615095517204,
"pooled_auc_strict": 0.9482228601031503,
"spec_top1": 0.875,
"spec_top3": 0.9583333333333334
},
"dev": {
"cat_map": 0.7612631638485738,
"cat_p5": 0.8,
"cat_p10": 0.7375,
"class_map": 0.6871950955434535,
"class_top3": 0.9354838709677419,
"tag_acc": 0.5967741935483871,
"tag_mrr": 0.7250974680232269,
"pooled_auc": 0.9690845183793936,
"pooled_auc_strict": 0.9550320284697509,
"spec_top1": 0.9,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7968612828098313,
"cat_p5": 0.775,
"cat_p10": 0.7374999999999999,
"class_map": 0.6471411183926385,
"class_top3": 0.8275862068965517,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6890846106364683,
"pooled_auc": 0.9563881781279087,
"pooled_auc_strict": 0.9393547439287676,
"spec_top1": 0.8571428571428571,
"spec_top3": 0.9047619047619048
}
}
],
"thresholds": {
"rule": "on the 219 tuning files (lenient labels), 0.5 z grid, top-down: strong = lowest t with LR(z >= t) >= 100; plausible = lowest t below it with band LR >= 20; weak = lowest t below that with band LR >= 5",
"grid_lenient": [
{
"z": 0.0,
"tp": 438,
"fp": 5459,
"recall": 0.9865,
"fpr": 0.41107,
"lr": 2.4,
"precision": 0.0743
},
{
"z": 0.25,
"tp": 431,
"fp": 4270,
"recall": 0.9707,
"fpr": 0.32154,
"lr": 3.0,
"precision": 0.0917
},
{
"z": 0.5,
"tp": 424,
"fp": 3314,
"recall": 0.955,
"fpr": 0.24955,
"lr": 3.8,
"precision": 0.1134
},
{
"z": 0.75,
"tp": 419,
"fp": 2477,
"recall": 0.9437,
"fpr": 0.18652,
"lr": 5.1,
"precision": 0.1447
},
{
"z": 1.0,
"tp": 411,
"fp": 1765,
"recall": 0.9257,
"fpr": 0.13291,
"lr": 7.0,
"precision": 0.1889
},
{
"z": 1.25,
"tp": 400,
"fp": 1214,
"recall": 0.9009,
"fpr": 0.09142,
"lr": 9.9,
"precision": 0.2478
},
{
"z": 1.5,
"tp": 388,
"fp": 853,
"recall": 0.8739,
"fpr": 0.06423,
"lr": 13.6,
"precision": 0.3127
},
{
"z": 1.75,
"tp": 380,
"fp": 567,
"recall": 0.8559,
"fpr": 0.0427,
"lr": 20.0,
"precision": 0.4013
},
{
"z": 2.0,
"tp": 365,
"fp": 359,
"recall": 0.8221,
"fpr": 0.02703,
"lr": 30.4,
"precision": 0.5041
},
{
"z": 2.25,
"tp": 345,
"fp": 235,
"recall": 0.777,
"fpr": 0.0177,
"lr": 43.9,
"precision": 0.5948
},
{
"z": 2.5,
"tp": 309,
"fp": 123,
"recall": 0.6959,
"fpr": 0.00926,
"lr": 75.1,
"precision": 0.7153
},
{
"z": 2.75,
"tp": 279,
"fp": 76,
"recall": 0.6284,
"fpr": 0.00572,
"lr": 109.8,
"precision": 0.7859
},
{
"z": 3.0,
"tp": 228,
"fp": 40,
"recall": 0.5135,
"fpr": 0.00301,
"lr": 170.5,
"precision": 0.8507
},
{
"z": 3.25,
"tp": 193,
"fp": 24,
"recall": 0.4347,
"fpr": 0.00181,
"lr": 240.5,
"precision": 0.8894
},
{
"z": 3.5,
"tp": 157,
"fp": 11,
"recall": 0.3536,
"fpr": 0.00083,
"lr": 426.9,
"precision": 0.9345
},
{
"z": 3.75,
"tp": 132,
"fp": 9,
"recall": 0.2973,
"fpr": 0.00068,
"lr": 438.7,
"precision": 0.9362
},
{
"z": 4.0,
"tp": 101,
"fp": 5,
"recall": 0.2275,
"fpr": 0.00038,
"lr": 604.2,
"precision": 0.9528
},
{
"z": 4.25,
"tp": 72,
"fp": 4,
"recall": 0.1622,
"fpr": 0.0003,
"lr": 538.4,
"precision": 0.9474
},
{
"z": 4.5,
"tp": 46,
"fp": 2,
"recall": 0.1036,
"fpr": 0.00015,
"lr": 687.9,
"precision": 0.9583
},
{
"z": 4.75,
"tp": 31,
"fp": 2,
"recall": 0.0698,
"fpr": 0.00015,
"lr": 463.6,
"precision": 0.9394
},
{
"z": 5.0,
"tp": 19,
"fp": 0,
"recall": 0.0428,
"fpr": 0.0,
"lr": null,
"precision": 1.0
},
{
"z": 5.25,
"tp": 9,
"fp": 0,
"recall": 0.0203,
"fpr": 0.0,
"lr": null,
"precision": 1.0
},
{
"z": 5.5,
"tp": 5,
"fp": 0,
"recall": 0.0113,
"fpr": 0.0,
"lr": null,
"precision": 1.0
},
{
"z": 5.75,
"tp": 1,
"fp": 0,
"recall": 0.0023,
"fpr": 0.0,
"lr": null,
"precision": 1.0
},
{
"z": 6.0,
"tp": 0,
"fp": 0,
"recall": 0.0,
"fpr": 0.0,
"lr": null,
"precision": 0.0
}
],
"grid_strict": [
{
"z": 0.0,
"tp": 438,
"fp": 6812,
"recall": 0.9865,
"fpr": 0.45761,
"lr": 2.2,
"precision": 0.0604
},
{
"z": 0.25,
"tp": 431,
"fp": 5548,
"recall": 0.9707,
"fpr": 0.3727,
"lr": 2.6,
"precision": 0.0721
},
{
"z": 0.5,
"tp": 424,
"fp": 4504,
"recall": 0.955,
"fpr": 0.30257,
"lr": 3.2,
"precision": 0.086
},
{
"z": 0.75,
"tp": 419,
"fp": 3548,
"recall": 0.9437,
"fpr": 0.23834,
"lr": 4.0,
"precision": 0.1056
},
{
"z": 1.0,
"tp": 411,
"fp": 2718,
"recall": 0.9257,
"fpr": 0.18259,
"lr": 5.1,
"precision": 0.1314
},
{
"z": 1.25,
"tp": 400,
"fp": 2056,
"recall": 0.9009,
"fpr": 0.13812,
"lr": 6.5,
"precision": 0.1629
},
{
"z": 1.5,
"tp": 388,
"fp": 1564,
"recall": 0.8739,
"fpr": 0.10507,
"lr": 8.3,
"precision": 0.1988
},
{
"z": 1.75,
"tp": 380,
"fp": 1174,
"recall": 0.8559,
"fpr": 0.07887,
"lr": 10.9,
"precision": 0.2445
},
{
"z": 2.0,
"tp": 365,
"fp": 870,
"recall": 0.8221,
"fpr": 0.05844,
"lr": 14.1,
"precision": 0.2955
},
{
"z": 2.25,
"tp": 345,
"fp": 645,
"recall": 0.777,
"fpr": 0.04333,
"lr": 17.9,
"precision": 0.3485
},
{
"z": 2.5,
"tp": 309,
"fp": 415,
"recall": 0.6959,
"fpr": 0.02788,
"lr": 25.0,
"precision": 0.4268
},
{
"z": 2.75,
"tp": 279,
"fp": 281,
"recall": 0.6284,
"fpr": 0.01888,
"lr": 33.3,
"precision": 0.4982
},
{
"z": 3.0,
"tp": 228,
"fp": 190,
"recall": 0.5135,
"fpr": 0.01276,
"lr": 40.2,
"precision": 0.5455
},
{
"z": 3.25,
"tp": 193,
"fp": 119,
"recall": 0.4347,
"fpr": 0.00799,
"lr": 54.4,
"precision": 0.6186
},
{
"z": 3.5,
"tp": 157,
"fp": 72,
"recall": 0.3536,
"fpr": 0.00484,
"lr": 73.1,
"precision": 0.6856
},
{
"z": 3.75,
"tp": 132,
"fp": 54,
"recall": 0.2973,
"fpr": 0.00363,
"lr": 82.0,
"precision": 0.7097
},
{
"z": 4.0,
"tp": 101,
"fp": 37,
"recall": 0.2275,
"fpr": 0.00249,
"lr": 91.5,
"precision": 0.7319
},
{
"z": 4.25,
"tp": 72,
"fp": 26,
"recall": 0.1622,
"fpr": 0.00175,
"lr": 92.8,
"precision": 0.7347
},
{
"z": 4.5,
"tp": 46,
"fp": 15,
"recall": 0.1036,
"fpr": 0.00101,
"lr": 102.8,
"precision": 0.7541
},
{
"z": 4.75,
"tp": 31,
"fp": 10,
"recall": 0.0698,
"fpr": 0.00067,
"lr": 103.9,
"precision": 0.7561
},
{
"z": 5.0,
"tp": 19,
"fp": 4,
"recall": 0.0428,
"fpr": 0.00027,
"lr": 159.3,
"precision": 0.8261
},
{
"z": 5.25,
"tp": 9,
"fp": 3,
"recall": 0.0203,
"fpr": 0.0002,
"lr": 100.6,
"precision": 0.75
},
{
"z": 5.5,
"tp": 5,
"fp": 1,
"recall": 0.0113,
"fpr": 7e-05,
"lr": 167.6,
"precision": 0.8333
},
{
"z": 5.75,
"tp": 1,
"fp": 1,
"recall": 0.0023,
"fpr": 7e-05,
"lr": 33.5,
"precision": 0.5
},
{
"z": 6.0,
"tp": 0,
"fp": 0,
"recall": 0.0,
"fpr": 0.0,
"lr": null,
"precision": 0.0
}
],
"picked": {
"strong": 3.0,
"plausible": 2.5,
"weak": 2.0
},
"shipped": {
"strong": 3.0,
"plausible": 2.5,
"weak": 2.0
},
"bands_tuning": {
"lenient": {
"n_pos": 444,
"n_neg": 13280,
"base_rate": 0.0324,
"strong": {
"z_range": [
3.0,
null
],
"tp": 228,
"fp": 40,
"lr": 170.49,
"precision": 0.851,
"precision_even_odds": 0.994,
"recall_at_or_above": 0.514
},
"plausible": {
"z_range": [
2.5,
3.0
],
"tp": 81,
"fp": 83,
"lr": 29.19,
"precision": 0.494,
"precision_even_odds": 0.967,
"recall_at_or_above": 0.696
},
"weak": {
"z_range": [
2.0,
2.5
],
"tp": 56,
"fp": 236,
"lr": 7.1,
"precision": 0.192,
"precision_even_odds": 0.877,
"recall_at_or_above": 0.822
},
"no match": {
"z_range": [
null,
2.0
],
"tp": 79,
"fp": 12921,
"lr": 0.18,
"precision": 0.006,
"precision_even_odds": 0.155,
"recall_at_or_above": 1.0
}
},
"strict": {
"n_pos": 444,
"n_neg": 14886,
"base_rate": 0.029,
"strong": {
"z_range": [
3.0,
null
],
"tp": 228,
"fp": 190,
"lr": 40.23,
"precision": 0.545,
"precision_even_odds": 0.976,
"recall_at_or_above": 0.514
},
"plausible": {
"z_range": [
2.5,
3.0
],
"tp": 81,
"fp": 225,
"lr": 12.07,
"precision": 0.265,
"precision_even_odds": 0.923,
"recall_at_or_above": 0.696
},
"weak": {
"z_range": [
2.0,
2.5
],
"tp": 56,
"fp": 455,
"lr": 4.13,
"precision": 0.11,
"precision_even_odds": 0.805,
"recall_at_or_above": 0.822
},
"no match": {
"z_range": [
null,
2.0
],
"tp": 79,
"fp": 14016,
"lr": 0.19,
"precision": 0.006,
"precision_even_odds": 0.159,
"recall_at_or_above": 1.0
}
}
}
},
"reference_sensitivity": {
"folder-balanced (used)": {
"dev": {
"cat_map": 0.777586606292167,
"cat_p5": 0.85,
"cat_p10": 0.75,
"class_map": 0.6779892152094823,
"class_top3": 0.8870967741935484,
"tag_acc": 0.6008064516129032,
"tag_mrr": 0.7313790616536111,
"pooled_auc": 0.9731773544405689,
"pooled_auc_strict": 0.9595833926453143,
"spec_top1": 0.9,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7932570388183038,
"cat_p5": 0.825,
"cat_p10": 0.7125,
"class_map": 0.6390066424489291,
"class_top3": 0.8793103448275862,
"tag_acc": 0.5526315789473685,
"tag_mrr": 0.6876431706014001,
"pooled_auc": 0.9558856317750858,
"pooled_auc_strict": 0.9393954636620294,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
}
},
"uniform over files": {
"dev": {
"cat_map": 0.7718379693024295,
"cat_p5": 0.85,
"cat_p10": 0.75,
"class_map": 0.6733486372216337,
"class_top3": 0.8870967741935484,
"tag_acc": 0.6048387096774194,
"tag_mrr": 0.7347743311927373,
"pooled_auc": 0.9716329487523477,
"pooled_auc_strict": 0.9580948991696323,
"spec_top1": 0.9,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7943198773256326,
"cat_p5": 0.825,
"cat_p10": 0.7125,
"class_map": 0.6435487807470288,
"class_top3": 0.896551724137931,
"tag_acc": 0.5421052631578948,
"tag_mrr": 0.6877697542124378,
"pooled_auc": 0.9566429903631428,
"pooled_auc_strict": 0.9400701337523474,
"spec_top1": 0.7619047619047619,
"spec_top3": 0.9047619047619048
}
},
"without music-like refs": {
"dev": {
"cat_map": 0.7799268851074638,
"cat_p5": 0.875,
"cat_p10": 0.7749999999999999,
"class_map": 0.6769617486987801,
"class_top3": 0.8870967741935484,
"tag_acc": 0.6169354838709677,
"tag_mrr": 0.7386258398859205,
"pooled_auc": 0.9738985779447277,
"pooled_auc_strict": 0.9605219454329774,
"spec_top1": 0.8,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7712286243312736,
"cat_p5": 0.8,
"cat_p10": 0.7125,
"class_map": 0.6346012267444308,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5631578947368421,
"tag_mrr": 0.6975672928451159,
"pooled_auc": 0.9592043841860696,
"pooled_auc_strict": 0.9430243104791834,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9523809523809523
}
},
"transductive (tuning set itself)": {
"dev": {
"cat_map": 0.7728621385713701,
"cat_p5": 0.875,
"cat_p10": 0.7625,
"class_map": 0.6770566888086154,
"class_top3": 0.9193548387096774,
"tag_acc": 0.6290322580645161,
"tag_mrr": 0.7471394689539851,
"pooled_auc": 0.9737123691977462,
"pooled_auc_strict": 0.9601024911032029,
"spec_top1": 0.85,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7940628661543965,
"cat_p5": 0.8,
"cat_p10": 0.7125,
"class_map": 0.6287646366614319,
"class_top3": 0.8448275862068966,
"tag_acc": 0.5736842105263158,
"tag_mrr": 0.7073285970723643,
"pooled_auc": 0.9612526144797052,
"pooled_auc_strict": 0.945261500530155,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
},
"no term bias": {
"dev": {
"cat_map": 0.7706841705015869,
"cat_p5": 0.8999999999999999,
"cat_p10": 0.7375,
"class_map": 0.6757453275548259,
"class_top3": 0.9032258064516129,
"tag_acc": 0.6209677419354839,
"tag_mrr": 0.7338462142037805,
"pooled_auc": 0.9712261872819963,
"pooled_auc_strict": 0.9565115065243179,
"spec_top1": 0.85,
"spec_top3": 1.0
},
"test": {
"cat_map": 0.7699222515537358,
"cat_p5": 0.8,
"cat_p10": 0.7125,
"class_map": 0.6454645266686625,
"class_top3": 0.8620689655172413,
"tag_acc": 0.5578947368421052,
"tag_mrr": 0.6902199626875211,
"pooled_auc": 0.9532366462462972,
"pooled_auc_strict": 0.9373035871689726,
"spec_top1": 0.8095238095238095,
"spec_top3": 0.9047619047619048
}
}
},
"term_bias": {
"highest": [
[
"a professionally mixed and mastered music track",
0.182
],
[
"a high quality studio recording",
0.172
],
[
"a glockenspiel",
0.124
],
[
"romantic music",
0.114
],
[
"upbeat corporate pop music",
0.113
],
[
"a dry close sound",
0.112
],
[
"a robotic synthetic sound",
0.11
],
[
"an enter key press",
0.107
],
[
"warm and cozy music",
0.106
],
[
"tech house music",
0.104
]
],
"lowest": [
[
"a man speaking",
-0.119
],
[
"people talking in the background",
-0.12
],
[
"a man shouting",
-0.124
],
[
"a crowd talking",
-0.125
],
[
"people laughing",
-0.13
]
]
},
"seconds": 66.7
}